com.google.cloud.spark.bigquery

Class SparkBigQueryConfig

java.lang.Object

com.google.cloud.spark.bigquery.SparkBigQueryConfig

All Implemented Interfaces:

BigQueryClient.CreateTableOptions, BigQueryClient.LoadDataOptions, BigQueryConfig, Serializable

public class SparkBigQueryConfig
extends Object
implements BigQueryConfig, BigQueryClient.CreateTableOptions, BigQueryClient.LoadDataOptions, Serializable

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	SparkBigQueryConfig.IntermediateFormat
static class	SparkBigQueryConfig.WriteMethod

Field Summary

Fields

Modifier and Type	Field and Description
static String	BIGQUERY_JOB_TIMEOUT_IN_MINUTES
static String	DATE_PARTITION_PARAM
static int	DEFAULT_CACHE_EXPIRATION_IN_MINUTES
static com.google.cloud.bigquery.QueryJobConfiguration.Priority	DEFAULT_JOB_PRIORITY
static String	ENABLE_LIST_INFERENCE
static String	GPN_ATTRIBUTION
static String	IMPERSONATION_FOR_GROUP_PREFIX
static String	IMPERSONATION_FOR_USER_PREFIX
static String	IMPERSONATION_GLOBAL
static String	INTERMEDIATE_FORMAT_OPTION
static int	MAX_TRACE_ID_LENGTH
static int	MIN_BUFFERED_RESPONSES_PER_STREAM
static int	MIN_STREAMS_PER_PARTITION
static String	partitionOverwriteModeProperty
PartitionOverwriteMode	partitionOverwriteModeValue
static String	USE_AVRO_LOGICAL_TYPES_OPTION
static String	VALIDATE_SPARK_AVRO_PARAM
static String	VIEWS_ENABLED_OPTION
static String	WRITE_AT_LEAST_ONCE_OPTION
static String	WRITE_METHOD_PARAM

Method Summary

Modifier and Type	Method and Description
com.google.auth.Credentials	createCredentials()
static SparkBigQueryConfig	from(Map<String,String> optionsInput, com.google.common.collect.ImmutableMap<String,String> originalGlobalOptions, org.apache.hadoop.conf.Configuration hadoopConfiguration, com.google.common.collect.ImmutableMap<String,String> customDefaults, int defaultParallelism, org.apache.spark.sql.internal.SQLConf sqlConf, String sparkVersion, Optional<org.apache.spark.sql.types.StructType> schema, boolean tableIsMandatory)
static SparkBigQueryConfig	from(Map<String,String> options, com.google.common.collect.ImmutableMap<String,String> customDefaults, DataSourceVersion dataSourceVersion, org.apache.spark.sql.SparkSession spark, Optional<org.apache.spark.sql.types.StructType> schema, boolean tableIsMandatory)
Optional<String>	getAccessToken()
Optional<String>	getAccessTokenProviderConfig()
Optional<String>	getAccessTokenProviderFQCN()
boolean	getAllowMapTypeConversion()
com.google.cloud.bigquery.storage.v1.ArrowSerializationOptions.CompressionCodec	getArrowCompressionCodec()
int	getBigQueryClientConnectTimeout()
int	getBigQueryClientReadTimeout()
com.google.api.gax.retrying.RetrySettings	getBigQueryClientRetrySettings()
com.google.api.gax.retrying.RetrySettings	getBigqueryDataWriteHelperRetrySettings()
Optional<String>	getBigQueryHttpEndpoint()
com.google.common.collect.ImmutableMap<String,String>	getBigQueryJobLabels()
long	getBigQueryJobTimeoutInMinutes()
BigQueryProxyConfig	getBigQueryProxyConfig()
Optional<String>	getBigQueryStorageGrpcEndpoint()
com.google.common.collect.ImmutableMap<String,String>	getBigQueryTableLabels()
int	getCacheExpirationTimeInMinutes()
int	getChannelPoolSize()
Optional<com.google.common.collect.ImmutableList<String>>	getClusteredFields()
Optional<com.google.cloud.bigquery.JobInfo.CreateDisposition>	getCreateDisposition()
Optional<Long>	getCreateReadSessionTimeoutInSeconds()
Optional<String>	getCredentialsFile()
Optional<String>	getCredentialsKey()
com.google.common.collect.ImmutableList<String>	getDecimalTargetTypes()
int	getDefaultParallelism()
boolean	getEnableModeCheckForSchemaFields()
Optional<String>	getFilter()
Optional<Integer>	getFlowControlWindowBytes()
Optional<String>	getGpn()
Optional<String>	getImpersonationServiceAccount()
Optional<Map<String,String>>	getImpersonationServiceAccountsForGroups()
Optional<Map<String,String>>	getImpersonationServiceAccountsForUsers()
SparkBigQueryConfig.IntermediateFormat	getIntermediateFormat()
Optional<String>	getKmsKeyName()
com.google.common.collect.ImmutableList<com.google.cloud.bigquery.JobInfo.SchemaUpdateOption>	getLoadSchemaUpdateOptions()
Set<String>	getLoggedInUserGroups()
String	getLoggedInUserName()
Optional<String>	getMaterializationDataset()
int	getMaterializationExpirationTimeInMinutes()
Optional<String>	getMaterializationProject()
OptionalInt	getMaxParallelism()
int	getMaxReadRowsRetries()
String	getParentProjectId()
OptionalLong	getPartitionExpirationMs()
Optional<String>	getPartitionField()
PartitionOverwriteMode	getPartitionOverwriteModeValue()
Optional<com.google.cloud.bigquery.RangePartitioning.Range>	getPartitionRange()
Optional<Boolean>	getPartitionRequireFilter()
Optional<com.google.cloud.bigquery.TimePartitioning.Type>	getPartitionType()
com.google.cloud.bigquery.TimePartitioning.Type	getPartitionTypeOrDefault()
Optional<String>	getPersistentGcsBucket()
Optional<String>	getPersistentGcsPath()
OptionalInt	getPreferredMinParallelism()
boolean	getPushAllFilters()
Optional<String>	getQuery()
com.google.cloud.bigquery.QueryJobConfiguration.Priority	getQueryJobPriority()
com.google.cloud.bigquery.storage.v1.DataFormat	getReadDataFormat()
com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions.ResponseCompressionCodec	getResponseCompressionCodec()
Optional<org.apache.spark.sql.types.StructType>	getSchema()
OptionalLong	getSnapshotTimeMillis()
com.google.cloud.bigquery.TableId	getTableId()
com.google.cloud.bigquery.TableId	getTableIdWithoutThePartition() Returns the table id, without the added partition id if it exists.
Optional<String>	getTemporaryGcsBucket()
Optional<String>	getTraceId()
SparkBigQueryConfig.WriteMethod	getWriteMethod()
boolean	isCombinePushedDownFilters()
boolean	isOptimizedEmptyProjection()
boolean	isUseAvroLogicalTypes()
boolean	isViewsEnabled()
boolean	isWriteAtLeastOnce()
ReadSessionCreatorConfig	toReadSessionCreatorConfig()
BigQueryClient.ReadTableOptions	toReadTableOptions()
boolean	useParentProjectForMetadataOperations()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface com.google.cloud.bigquery.connector.common.BigQueryConfig

areClientCreationConfigsEqual, getClientCreationHashCode

Methods inherited from interface com.google.cloud.bigquery.connector.common.BigQueryClient.CreateTableOptions

of

Field Detail

MAX_TRACE_ID_LENGTH

public static final int MAX_TRACE_ID_LENGTH

See Also:

Constant Field Values

IMPERSONATION_GLOBAL

public static final String IMPERSONATION_GLOBAL

See Also:

Constant Field Values

IMPERSONATION_FOR_USER_PREFIX

public static final String IMPERSONATION_FOR_USER_PREFIX

See Also:

Constant Field Values

IMPERSONATION_FOR_GROUP_PREFIX

public static final String IMPERSONATION_FOR_GROUP_PREFIX

See Also:

Constant Field Values

VIEWS_ENABLED_OPTION

public static final String VIEWS_ENABLED_OPTION

See Also:

Constant Field Values

USE_AVRO_LOGICAL_TYPES_OPTION

public static final String USE_AVRO_LOGICAL_TYPES_OPTION

See Also:

Constant Field Values

DATE_PARTITION_PARAM

public static final String DATE_PARTITION_PARAM

See Also:

Constant Field Values

VALIDATE_SPARK_AVRO_PARAM

public static final String VALIDATE_SPARK_AVRO_PARAM

See Also:

Constant Field Values

ENABLE_LIST_INFERENCE

public static final String ENABLE_LIST_INFERENCE

See Also:

Constant Field Values

INTERMEDIATE_FORMAT_OPTION

public static final String INTERMEDIATE_FORMAT_OPTION

See Also:

Constant Field Values

WRITE_METHOD_PARAM

public static final String WRITE_METHOD_PARAM

See Also:

Constant Field Values

WRITE_AT_LEAST_ONCE_OPTION

public static final String WRITE_AT_LEAST_ONCE_OPTION

See Also:

Constant Field Values

MIN_BUFFERED_RESPONSES_PER_STREAM

public static final int MIN_BUFFERED_RESPONSES_PER_STREAM

See Also:

Constant Field Values

MIN_STREAMS_PER_PARTITION

public static final int MIN_STREAMS_PER_PARTITION

See Also:

Constant Field Values

DEFAULT_CACHE_EXPIRATION_IN_MINUTES

public static final int DEFAULT_CACHE_EXPIRATION_IN_MINUTES

See Also:

Constant Field Values

DEFAULT_JOB_PRIORITY

public static final com.google.cloud.bigquery.QueryJobConfiguration.Priority DEFAULT_JOB_PRIORITY

partitionOverwriteModeProperty

public static final String partitionOverwriteModeProperty

See Also:

Constant Field Values

partitionOverwriteModeValue

public PartitionOverwriteMode partitionOverwriteModeValue

BIGQUERY_JOB_TIMEOUT_IN_MINUTES

public static final String BIGQUERY_JOB_TIMEOUT_IN_MINUTES

See Also:

Constant Field Values

GPN_ATTRIBUTION

public static final String GPN_ATTRIBUTION

See Also:

Constant Field Values

Method Detail

from

public static SparkBigQueryConfig from(Map<String,String> options,
                                       com.google.common.collect.ImmutableMap<String,String> customDefaults,
                                       DataSourceVersion dataSourceVersion,
                                       org.apache.spark.sql.SparkSession spark,
                                       Optional<org.apache.spark.sql.types.StructType> schema,
                                       boolean tableIsMandatory)

from

public static SparkBigQueryConfig from(Map<String,String> optionsInput,
                                       com.google.common.collect.ImmutableMap<String,String> originalGlobalOptions,
                                       org.apache.hadoop.conf.Configuration hadoopConfiguration,
                                       com.google.common.collect.ImmutableMap<String,String> customDefaults,
                                       int defaultParallelism,
                                       org.apache.spark.sql.internal.SQLConf sqlConf,
                                       String sparkVersion,
                                       Optional<org.apache.spark.sql.types.StructType> schema,
                                       boolean tableIsMandatory)

createCredentials

public com.google.auth.Credentials createCredentials()

getTableId

public com.google.cloud.bigquery.TableId getTableId()

Specified by:

getTableId in interface BigQueryClient.LoadDataOptions

getTableIdWithoutThePartition

public com.google.cloud.bigquery.TableId getTableIdWithoutThePartition()

Returns the table id, without the added partition id if it exists.

getQuery

public Optional<String> getQuery()

getParentProjectId

public String getParentProjectId()

Specified by:

getParentProjectId in interface BigQueryConfig

useParentProjectForMetadataOperations

public boolean useParentProjectForMetadataOperations()

Specified by:

useParentProjectForMetadataOperations in interface BigQueryConfig

getAccessTokenProviderFQCN

public Optional<String> getAccessTokenProviderFQCN()

Specified by:

getAccessTokenProviderFQCN in interface BigQueryConfig

getAccessTokenProviderConfig

public Optional<String> getAccessTokenProviderConfig()

Specified by:

getAccessTokenProviderConfig in interface BigQueryConfig

getLoggedInUserName

public String getLoggedInUserName()

Specified by:

getLoggedInUserName in interface BigQueryConfig

getLoggedInUserGroups

public Set<String> getLoggedInUserGroups()

Specified by:

getLoggedInUserGroups in interface BigQueryConfig

getImpersonationServiceAccountsForUsers

public Optional<Map<String,String>> getImpersonationServiceAccountsForUsers()

Specified by:

getImpersonationServiceAccountsForUsers in interface BigQueryConfig

getImpersonationServiceAccountsForGroups

public Optional<Map<String,String>> getImpersonationServiceAccountsForGroups()

Specified by:

getImpersonationServiceAccountsForGroups in interface BigQueryConfig

getImpersonationServiceAccount

public Optional<String> getImpersonationServiceAccount()

Specified by:

getImpersonationServiceAccount in interface BigQueryConfig

getCredentialsKey

public Optional<String> getCredentialsKey()

Specified by:

getCredentialsKey in interface BigQueryConfig

getCredentialsFile

public Optional<String> getCredentialsFile()

Specified by:

getCredentialsFile in interface BigQueryConfig

getAccessToken

public Optional<String> getAccessToken()

Specified by:

getAccessToken in interface BigQueryConfig

getFilter

public Optional<String> getFilter()

getSchema

public Optional<org.apache.spark.sql.types.StructType> getSchema()

getMaxParallelism

public OptionalInt getMaxParallelism()

getPreferredMinParallelism

public OptionalInt getPreferredMinParallelism()

getDefaultParallelism

public int getDefaultParallelism()

getTemporaryGcsBucket

public Optional<String> getTemporaryGcsBucket()

getPersistentGcsBucket

public Optional<String> getPersistentGcsBucket()

getPersistentGcsPath

public Optional<String> getPersistentGcsPath()

getIntermediateFormat

public SparkBigQueryConfig.IntermediateFormat getIntermediateFormat()

getReadDataFormat

public com.google.cloud.bigquery.storage.v1.DataFormat getReadDataFormat()

getArrowCompressionCodec

public com.google.cloud.bigquery.storage.v1.ArrowSerializationOptions.CompressionCodec getArrowCompressionCodec()

getResponseCompressionCodec

public com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions.ResponseCompressionCodec getResponseCompressionCodec()

isCombinePushedDownFilters

public boolean isCombinePushedDownFilters()

isUseAvroLogicalTypes

public boolean isUseAvroLogicalTypes()

Specified by:

isUseAvroLogicalTypes in interface BigQueryClient.LoadDataOptions

getDecimalTargetTypes

public com.google.common.collect.ImmutableList<String> getDecimalTargetTypes()

Specified by:

getDecimalTargetTypes in interface BigQueryClient.LoadDataOptions

isViewsEnabled

public boolean isViewsEnabled()

Specified by:

isViewsEnabled in interface BigQueryConfig

getMaterializationProject

public Optional<String> getMaterializationProject()

Specified by:

getMaterializationProject in interface BigQueryConfig

getMaterializationDataset

public Optional<String> getMaterializationDataset()

Specified by:

getMaterializationDataset in interface BigQueryConfig

getPartitionField

public Optional<String> getPartitionField()

Specified by:

getPartitionField in interface BigQueryClient.LoadDataOptions

getPartitionExpirationMs

public OptionalLong getPartitionExpirationMs()

Specified by:

getPartitionExpirationMs in interface BigQueryClient.LoadDataOptions

getPartitionRequireFilter

public Optional<Boolean> getPartitionRequireFilter()

Specified by:

getPartitionRequireFilter in interface BigQueryClient.LoadDataOptions

getPartitionType

public Optional<com.google.cloud.bigquery.TimePartitioning.Type> getPartitionType()

Specified by:

getPartitionType in interface BigQueryClient.LoadDataOptions

getPartitionRange

public Optional<com.google.cloud.bigquery.RangePartitioning.Range> getPartitionRange()

Specified by:

getPartitionRange in interface BigQueryClient.LoadDataOptions

getPartitionTypeOrDefault

public com.google.cloud.bigquery.TimePartitioning.Type getPartitionTypeOrDefault()

Specified by:

getPartitionTypeOrDefault in interface BigQueryClient.LoadDataOptions

getClusteredFields

public Optional<com.google.common.collect.ImmutableList<String>> getClusteredFields()

Specified by:

getClusteredFields in interface BigQueryClient.LoadDataOptions

getCreateDisposition

public Optional<com.google.cloud.bigquery.JobInfo.CreateDisposition> getCreateDisposition()

Specified by:

getCreateDisposition in interface BigQueryClient.LoadDataOptions

isOptimizedEmptyProjection

public boolean isOptimizedEmptyProjection()

getLoadSchemaUpdateOptions

public com.google.common.collect.ImmutableList<com.google.cloud.bigquery.JobInfo.SchemaUpdateOption> getLoadSchemaUpdateOptions()

Specified by:

getLoadSchemaUpdateOptions in interface BigQueryClient.LoadDataOptions

getMaterializationExpirationTimeInMinutes

public int getMaterializationExpirationTimeInMinutes()

getMaxReadRowsRetries

public int getMaxReadRowsRetries()

getPushAllFilters

public boolean getPushAllFilters()

getEnableModeCheckForSchemaFields

public boolean getEnableModeCheckForSchemaFields()

Specified by:

getEnableModeCheckForSchemaFields in interface BigQueryClient.LoadDataOptions

getBigQueryClientConnectTimeout

public int getBigQueryClientConnectTimeout()

Specified by:

getBigQueryClientConnectTimeout in interface BigQueryConfig

getBigQueryClientReadTimeout

public int getBigQueryClientReadTimeout()

Specified by:

getBigQueryClientReadTimeout in interface BigQueryConfig

getBigQueryProxyConfig

public BigQueryProxyConfig getBigQueryProxyConfig()

Specified by:

getBigQueryProxyConfig in interface BigQueryConfig

getBigQueryStorageGrpcEndpoint

public Optional<String> getBigQueryStorageGrpcEndpoint()

Specified by:

getBigQueryStorageGrpcEndpoint in interface BigQueryConfig

getBigQueryHttpEndpoint

public Optional<String> getBigQueryHttpEndpoint()

Specified by:

getBigQueryHttpEndpoint in interface BigQueryConfig

getCacheExpirationTimeInMinutes

public int getCacheExpirationTimeInMinutes()

Specified by:

getCacheExpirationTimeInMinutes in interface BigQueryConfig

getCreateReadSessionTimeoutInSeconds

public Optional<Long> getCreateReadSessionTimeoutInSeconds()

Specified by:

getCreateReadSessionTimeoutInSeconds in interface BigQueryConfig

getPartitionOverwriteModeValue

public PartitionOverwriteMode getPartitionOverwriteModeValue()

getChannelPoolSize

public int getChannelPoolSize()

Specified by:

getChannelPoolSize in interface BigQueryConfig

getFlowControlWindowBytes

public Optional<Integer> getFlowControlWindowBytes()

Specified by:

getFlowControlWindowBytes in interface BigQueryConfig

getQueryJobPriority

public com.google.cloud.bigquery.QueryJobConfiguration.Priority getQueryJobPriority()

Specified by:

getQueryJobPriority in interface BigQueryConfig

getKmsKeyName

public Optional<String> getKmsKeyName()

Specified by:

getKmsKeyName in interface BigQueryClient.CreateTableOptions

Specified by:

getKmsKeyName in interface BigQueryClient.LoadDataOptions

getBigQueryClientRetrySettings

public com.google.api.gax.retrying.RetrySettings getBigQueryClientRetrySettings()

Specified by:

getBigQueryClientRetrySettings in interface BigQueryConfig

getBigqueryDataWriteHelperRetrySettings

public com.google.api.gax.retrying.RetrySettings getBigqueryDataWriteHelperRetrySettings()

getWriteMethod

public SparkBigQueryConfig.WriteMethod getWriteMethod()

isWriteAtLeastOnce

public boolean isWriteAtLeastOnce()

getTraceId

public Optional<String> getTraceId()

getBigQueryJobLabels

public com.google.common.collect.ImmutableMap<String,String> getBigQueryJobLabels()

Specified by:

getBigQueryJobLabels in interface BigQueryConfig

getAllowMapTypeConversion

public boolean getAllowMapTypeConversion()

getBigQueryJobTimeoutInMinutes

public long getBigQueryJobTimeoutInMinutes()

Specified by:

getBigQueryJobTimeoutInMinutes in interface BigQueryConfig

getBigQueryTableLabels

public com.google.common.collect.ImmutableMap<String,String> getBigQueryTableLabels()

Specified by:

getBigQueryTableLabels in interface BigQueryClient.CreateTableOptions

getGpn

public Optional<String> getGpn()

getSnapshotTimeMillis

public OptionalLong getSnapshotTimeMillis()

toReadSessionCreatorConfig

public ReadSessionCreatorConfig toReadSessionCreatorConfig()

toReadTableOptions

public BigQueryClient.ReadTableOptions toReadTableOptions()