org.apache.beam.sdk.io.gcp.bigquery

Class StorageApiLoads<DestinationT,ElementT>

java.lang.Object

org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>>,WriteResult>

org.apache.beam.sdk.io.gcp.bigquery.StorageApiLoads<DestinationT,ElementT>

All Implemented Interfaces:

java.io.Serializable, org.apache.beam.sdk.transforms.display.HasDisplayData

public class StorageApiLoads<DestinationT,ElementT>
extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>>,WriteResult>

This PTransform manages loads into BigQuery using the Storage API.

See Also:

Serialized Form

Field Summary

Fields inherited from class org.apache.beam.sdk.transforms.PTransform

name, resourceHints

Constructor Summary

Constructors

Constructor and Description

StorageApiLoads(org.apache.beam.sdk.coders.Coder<DestinationT> destinationCoder, org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations<ElementT,DestinationT> dynamicDestinations, org.apache.beam.sdk.transforms.SerializableFunction<ElementT,RowMutationInformation> rowUpdateFn, BigQueryIO.Write.CreateDisposition createDisposition, java.lang.String kmsKey, org.joda.time.Duration triggeringFrequency, BigQueryServices bqServices, int numShards, boolean allowInconsistentWrites, boolean allowAutosharding, boolean autoUpdateSchema, boolean ignoreUnknownValues, boolean propagateSuccessfulStorageApiWrites, boolean usesCdc)

Method Summary

Modifier and Type	Method and Description
WriteResult	expand(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input)
WriteResult	expandInconsistent(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input, org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)
WriteResult	expandTriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input, org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder, org.apache.beam.sdk.coders.Coder<StorageApiWritePayload> payloadCoder)
WriteResult	expandUntriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input, org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)
org.apache.beam.sdk.values.TupleTag<BigQueryStorageApiInsertError>	getFailedRowsTag()

Methods inherited from class org.apache.beam.sdk.transforms.PTransform

compose, compose, getAdditionalInputs, getDefaultOutputCoder, getDefaultOutputCoder, getDefaultOutputCoder, getKindString, getName, getResourceHints, populateDisplayData, setResourceHints, toString, validate, validate

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Constructor Detail

StorageApiLoads

public StorageApiLoads(org.apache.beam.sdk.coders.Coder<DestinationT> destinationCoder,
                       org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations<ElementT,DestinationT> dynamicDestinations,
                       @Nullable
                       org.apache.beam.sdk.transforms.SerializableFunction<ElementT,RowMutationInformation> rowUpdateFn,
                       BigQueryIO.Write.CreateDisposition createDisposition,
                       java.lang.String kmsKey,
                       org.joda.time.Duration triggeringFrequency,
                       BigQueryServices bqServices,
                       int numShards,
                       boolean allowInconsistentWrites,
                       boolean allowAutosharding,
                       boolean autoUpdateSchema,
                       boolean ignoreUnknownValues,
                       boolean propagateSuccessfulStorageApiWrites,
                       boolean usesCdc)

Method Detail

getFailedRowsTag

public org.apache.beam.sdk.values.TupleTag<BigQueryStorageApiInsertError> getFailedRowsTag()

expand

public WriteResult expand(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input)

Specified by:

expand in class org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>>,WriteResult>

expandInconsistent

public WriteResult expandInconsistent(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input,
                                      org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)

expandTriggered

public WriteResult expandTriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input,
                                   org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder,
                                   org.apache.beam.sdk.coders.Coder<StorageApiWritePayload> payloadCoder)

expandUntriggered

public WriteResult expandUntriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input,
                                     org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)