org.apache.hudi.config

Class HoodieWriteConfig.Builder

java.lang.Object

org.apache.hudi.config.HoodieWriteConfig.Builder

Enclosing class:

HoodieWriteConfig

public static class HoodieWriteConfig.Builder
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
protected EngineType	engineType
protected HoodieWriteConfig	writeConfig

Constructor Summary

Constructors

Constructor and Description
Builder()

Method Summary

Modifier and Type	Method and Description
HoodieWriteConfig	build()
HoodieWriteConfig	build(boolean shouldValidate)
HoodieWriteConfig.Builder	combineDeleteInput(boolean onDelete)
HoodieWriteConfig.Builder	combineInput(boolean onInsert, boolean onUpsert)
HoodieWriteConfig.Builder	doSkipDefaultPartitionValidation(boolean skipDefaultPartitionValidation)
HoodieWriteConfig.Builder	forTable(String tableName)
HoodieWriteConfig.Builder	fromFile(File propertiesFile)
HoodieWriteConfig.Builder	fromInputStream(InputStream inputStream)
protected void	setDefaults()
HoodieWriteConfig.Builder	withAllowAutoEvolutionColumnDrop(boolean shouldAllowDroppedColumns)
HoodieWriteConfig.Builder	withAllowMultiWriteOnSameInstant(boolean allow)
HoodieWriteConfig.Builder	withAllowOperationMetadataField(boolean allowOperationMetadataField)
HoodieWriteConfig.Builder	withArchivalConfig(HoodieArchivalConfig cleanConfig)
HoodieWriteConfig.Builder	withAsyncConflictDetectorInitialDelayMs(long intervalMs)
HoodieWriteConfig.Builder	withAsyncConflictDetectorPeriodMs(long periodMs)
HoodieWriteConfig.Builder	withAutoAdjustLockConfigs(boolean autoAdjustLockConfigs)
HoodieWriteConfig.Builder	withAutoCommit(boolean autoCommit)
HoodieWriteConfig.Builder	withAvroSchemaValidate(boolean enable)
HoodieWriteConfig.Builder	withBootstrapConfig(HoodieBootstrapConfig bootstrapConfig)
HoodieWriteConfig.Builder	withBulkInsertParallelism(int bulkInsertParallelism)
HoodieWriteConfig.Builder	withBulkInsertSortMode(String mode)
HoodieWriteConfig.Builder	withCallbackConfig(HoodieWriteCommitCallbackConfig callbackConfig)
HoodieWriteConfig.Builder	withCleanConfig(HoodieCleanConfig cleanConfig)
HoodieWriteConfig.Builder	withClientInitCallbackClassNames(String classNames)
HoodieWriteConfig.Builder	withClusteringConfig(HoodieClusteringConfig clusteringConfig)
HoodieWriteConfig.Builder	withCompactionConfig(HoodieCompactionConfig compactionConfig)
HoodieWriteConfig.Builder	withConsistencyGuardConfig(ConsistencyGuardConfig consistencyGuardConfig)
HoodieWriteConfig.Builder	withDeleteParallelism(int parallelism)
HoodieWriteConfig.Builder	withEarlyConflictDetectionCheckCommitConflict(boolean enable)
HoodieWriteConfig.Builder	withEarlyConflictDetectionEnable(boolean enable)
HoodieWriteConfig.Builder	withEarlyConflictDetectionStrategy(String className)
HoodieWriteConfig.Builder	withEmbeddedTimelineServerEnabled(boolean enabled)
HoodieWriteConfig.Builder	withEmbeddedTimelineServerPort(int port)
HoodieWriteConfig.Builder	withEmbeddedTimelineServerReuseEnabled(boolean enabled)
HoodieWriteConfig.Builder	withEngineType(EngineType engineType)
HoodieWriteConfig.Builder	withExecutorType(String executorClass)
HoodieWriteConfig.Builder	withExternalSchemaTrasformation(boolean enabled)
HoodieWriteConfig.Builder	withFailureOnInlineTableServiceException(boolean fail)
HoodieWriteConfig.Builder	withFileIdPrefixProviderClassName(String fileIdPrefixProviderClassName)
HoodieWriteConfig.Builder	withFileSystemViewConfig(FileSystemViewStorageConfig viewStorageConfig)
HoodieWriteConfig.Builder	withFinalizeWriteParallelism(int parallelism)
HoodieWriteConfig.Builder	withHeartbeatIntervalInMs(Integer heartbeatIntervalInMs)
HoodieWriteConfig.Builder	withHeartbeatTolerableMisses(Integer heartbeatTolerableMisses)
HoodieWriteConfig.Builder	withHiveStylePartitioningEnabled(boolean enabled)
HoodieWriteConfig.Builder	withIndexConfig(HoodieIndexConfig indexConfig)
HoodieWriteConfig.Builder	withInternalSchemaCacheEnable(boolean enable)
HoodieWriteConfig.Builder	withKeyGenerator(String keyGeneratorClass)
HoodieWriteConfig.Builder	withLayoutConfig(HoodieLayoutConfig layoutConfig)
HoodieWriteConfig.Builder	withLockConfig(HoodieLockConfig lockConfig)
HoodieWriteConfig.Builder	withMarkersDeleteParallelism(int parallelism)
HoodieWriteConfig.Builder	withMarkersTimelineServerBasedBatchIntervalMs(long intervalMs)
HoodieWriteConfig.Builder	withMarkersTimelineServerBasedBatchNumThreads(int numThreads)
HoodieWriteConfig.Builder	withMarkersType(String markerType)
HoodieWriteConfig.Builder	withMemoryConfig(HoodieMemoryConfig memoryConfig)
HoodieWriteConfig.Builder	withMergeAllowDuplicateOnInserts(boolean routeInsertsToNewFiles)
HoodieWriteConfig.Builder	withMergeDataValidationCheckEnabled(boolean enabled)
HoodieWriteConfig.Builder	withMergeSmallFileGroupCandidatesLimit(int limit)
HoodieWriteConfig.Builder	withMetadataConfig(HoodieMetadataConfig metadataConfig)
HoodieWriteConfig.Builder	withMetricsConfig(HoodieMetricsConfig metricsConfig)
HoodieWriteConfig.Builder	withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig mericsGraphiteConfig)
HoodieWriteConfig.Builder	withMetricsJmxConfig(HoodieMetricsJmxConfig metricsJmxConfig)
HoodieWriteConfig.Builder	withParallelism(int insertShuffleParallelism, int upsertShuffleParallelism)
HoodieWriteConfig.Builder	withPath(String basePath)
HoodieWriteConfig.Builder	withPayloadConfig(HoodiePayloadConfig payloadConfig)
HoodieWriteConfig.Builder	withPopulateMetaFields(boolean populateMetaFields)
HoodieWriteConfig.Builder	withPreCombineField(String preCombineField)
HoodieWriteConfig.Builder	withPreCommitValidatorConfig(HoodiePreCommitValidatorConfig validatorConfig)
HoodieWriteConfig.Builder	withProperties(Properties properties)
HoodieWriteConfig.Builder	withProps(Map kvprops)
HoodieWriteConfig.Builder	withRecordMergerImpls(String recordMergerImpls)
HoodieWriteConfig.Builder	withRecordMergerStrategy(String recordMergerStrategy)
HoodieWriteConfig.Builder	withReleaseResourceEnabled(boolean enabled)
HoodieWriteConfig.Builder	withRollbackBackupDirectory(String backupDir)
HoodieWriteConfig.Builder	withRollbackBackupEnabled(boolean rollbackBackupEnabled)
HoodieWriteConfig.Builder	withRollbackParallelism(int rollbackParallelism)
HoodieWriteConfig.Builder	withRollbackUsingMarkers(boolean rollbackUsingMarkers)
HoodieWriteConfig.Builder	withSchema(String schemaStr)
HoodieWriteConfig.Builder	withSchemaEvolutionEnable(boolean enable)
HoodieWriteConfig.Builder	withStorageConfig(HoodieStorageConfig storageConfig)
HoodieWriteConfig.Builder	withTableServicesEnabled(boolean enabled)
HoodieWriteConfig.Builder	withTimelineLayoutVersion(int version)
HoodieWriteConfig.Builder	withUserDefinedBulkInsertPartitionerClass(String className)
HoodieWriteConfig.Builder	withUserDefinedBulkInsertPartitionerSortColumns(String columns)
HoodieWriteConfig.Builder	withWriteBufferLimitBytes(int writeBufferLimit)
HoodieWriteConfig.Builder	withWriteConcurrencyMode(WriteConcurrencyMode concurrencyMode)
HoodieWriteConfig.Builder	withWriteExecutorDisruptorWaitStrategy(String waitStrategy)
HoodieWriteConfig.Builder	withWriteExecutorDisruptorWriteBufferLimitBytes(long size)
HoodieWriteConfig.Builder	withWritePayLoad(String payload)
HoodieWriteConfig.Builder	withWritesFileIdEncoding(Integer fileIdEncoding)
HoodieWriteConfig.Builder	withWriteStatusClass(Class<? extends WriteStatus> writeStatusClass)
HoodieWriteConfig.Builder	withWriteStatusStorageLevel(String level)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

writeConfig

protected final HoodieWriteConfig writeConfig

engineType

protected EngineType engineType

Constructor Detail

Builder

public Builder()

Method Detail

withEngineType

public HoodieWriteConfig.Builder withEngineType(EngineType engineType)

fromFile

public HoodieWriteConfig.Builder fromFile(File propertiesFile)
                                   throws IOException

Throws:

IOException

fromInputStream

public HoodieWriteConfig.Builder fromInputStream(InputStream inputStream)
                                          throws IOException

Throws:

IOException

withProps

public HoodieWriteConfig.Builder withProps(Map kvprops)

withPath

public HoodieWriteConfig.Builder withPath(String basePath)

withSchema

public HoodieWriteConfig.Builder withSchema(String schemaStr)

withSchemaEvolutionEnable

public HoodieWriteConfig.Builder withSchemaEvolutionEnable(boolean enable)

withInternalSchemaCacheEnable

public HoodieWriteConfig.Builder withInternalSchemaCacheEnable(boolean enable)

withAvroSchemaValidate

public HoodieWriteConfig.Builder withAvroSchemaValidate(boolean enable)

withAllowAutoEvolutionColumnDrop

public HoodieWriteConfig.Builder withAllowAutoEvolutionColumnDrop(boolean shouldAllowDroppedColumns)

forTable

public HoodieWriteConfig.Builder forTable(String tableName)

withPreCombineField

public HoodieWriteConfig.Builder withPreCombineField(String preCombineField)

withWritePayLoad

public HoodieWriteConfig.Builder withWritePayLoad(String payload)

withRecordMergerImpls

public HoodieWriteConfig.Builder withRecordMergerImpls(String recordMergerImpls)

withRecordMergerStrategy

public HoodieWriteConfig.Builder withRecordMergerStrategy(String recordMergerStrategy)

withKeyGenerator

public HoodieWriteConfig.Builder withKeyGenerator(String keyGeneratorClass)

withExecutorType

public HoodieWriteConfig.Builder withExecutorType(String executorClass)

withTimelineLayoutVersion

public HoodieWriteConfig.Builder withTimelineLayoutVersion(int version)

withBulkInsertParallelism

public HoodieWriteConfig.Builder withBulkInsertParallelism(int bulkInsertParallelism)

withUserDefinedBulkInsertPartitionerClass

public HoodieWriteConfig.Builder withUserDefinedBulkInsertPartitionerClass(String className)

withUserDefinedBulkInsertPartitionerSortColumns

public HoodieWriteConfig.Builder withUserDefinedBulkInsertPartitionerSortColumns(String columns)

withDeleteParallelism

public HoodieWriteConfig.Builder withDeleteParallelism(int parallelism)

withFailureOnInlineTableServiceException

public HoodieWriteConfig.Builder withFailureOnInlineTableServiceException(boolean fail)

withParallelism

public HoodieWriteConfig.Builder withParallelism(int insertShuffleParallelism,
                                                 int upsertShuffleParallelism)

withRollbackParallelism

public HoodieWriteConfig.Builder withRollbackParallelism(int rollbackParallelism)

withRollbackUsingMarkers

public HoodieWriteConfig.Builder withRollbackUsingMarkers(boolean rollbackUsingMarkers)

withWriteBufferLimitBytes

public HoodieWriteConfig.Builder withWriteBufferLimitBytes(int writeBufferLimit)

withWriteExecutorDisruptorWaitStrategy

public HoodieWriteConfig.Builder withWriteExecutorDisruptorWaitStrategy(String waitStrategy)

withWriteExecutorDisruptorWriteBufferLimitBytes

public HoodieWriteConfig.Builder withWriteExecutorDisruptorWriteBufferLimitBytes(long size)

combineInput

public HoodieWriteConfig.Builder combineInput(boolean onInsert,
                                              boolean onUpsert)

combineDeleteInput

public HoodieWriteConfig.Builder combineDeleteInput(boolean onDelete)

withWriteStatusStorageLevel

public HoodieWriteConfig.Builder withWriteStatusStorageLevel(String level)

withIndexConfig

public HoodieWriteConfig.Builder withIndexConfig(HoodieIndexConfig indexConfig)

withStorageConfig

public HoodieWriteConfig.Builder withStorageConfig(HoodieStorageConfig storageConfig)

withCompactionConfig

public HoodieWriteConfig.Builder withCompactionConfig(HoodieCompactionConfig compactionConfig)

withCleanConfig

public HoodieWriteConfig.Builder withCleanConfig(HoodieCleanConfig cleanConfig)

withArchivalConfig

public HoodieWriteConfig.Builder withArchivalConfig(HoodieArchivalConfig cleanConfig)

withClusteringConfig

public HoodieWriteConfig.Builder withClusteringConfig(HoodieClusteringConfig clusteringConfig)

withLockConfig

public HoodieWriteConfig.Builder withLockConfig(HoodieLockConfig lockConfig)

withMetricsJmxConfig

public HoodieWriteConfig.Builder withMetricsJmxConfig(HoodieMetricsJmxConfig metricsJmxConfig)

withMetricsGraphiteConfig

public HoodieWriteConfig.Builder withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig mericsGraphiteConfig)

withPreCommitValidatorConfig

public HoodieWriteConfig.Builder withPreCommitValidatorConfig(HoodiePreCommitValidatorConfig validatorConfig)

withMetricsConfig

public HoodieWriteConfig.Builder withMetricsConfig(HoodieMetricsConfig metricsConfig)

withMemoryConfig

public HoodieWriteConfig.Builder withMemoryConfig(HoodieMemoryConfig memoryConfig)

withBootstrapConfig

public HoodieWriteConfig.Builder withBootstrapConfig(HoodieBootstrapConfig bootstrapConfig)

withPayloadConfig

public HoodieWriteConfig.Builder withPayloadConfig(HoodiePayloadConfig payloadConfig)

withMetadataConfig

public HoodieWriteConfig.Builder withMetadataConfig(HoodieMetadataConfig metadataConfig)

withAutoCommit

public HoodieWriteConfig.Builder withAutoCommit(boolean autoCommit)

withWriteStatusClass

public HoodieWriteConfig.Builder withWriteStatusClass(Class<? extends WriteStatus> writeStatusClass)

withFileSystemViewConfig

public HoodieWriteConfig.Builder withFileSystemViewConfig(FileSystemViewStorageConfig viewStorageConfig)

withConsistencyGuardConfig

public HoodieWriteConfig.Builder withConsistencyGuardConfig(ConsistencyGuardConfig consistencyGuardConfig)

withCallbackConfig

public HoodieWriteConfig.Builder withCallbackConfig(HoodieWriteCommitCallbackConfig callbackConfig)

withLayoutConfig

public HoodieWriteConfig.Builder withLayoutConfig(HoodieLayoutConfig layoutConfig)

withFinalizeWriteParallelism

public HoodieWriteConfig.Builder withFinalizeWriteParallelism(int parallelism)

withMarkersType

public HoodieWriteConfig.Builder withMarkersType(String markerType)

withMarkersTimelineServerBasedBatchNumThreads

public HoodieWriteConfig.Builder withMarkersTimelineServerBasedBatchNumThreads(int numThreads)

withMarkersTimelineServerBasedBatchIntervalMs

public HoodieWriteConfig.Builder withMarkersTimelineServerBasedBatchIntervalMs(long intervalMs)

withMarkersDeleteParallelism

public HoodieWriteConfig.Builder withMarkersDeleteParallelism(int parallelism)

withEmbeddedTimelineServerEnabled

public HoodieWriteConfig.Builder withEmbeddedTimelineServerEnabled(boolean enabled)

withEmbeddedTimelineServerReuseEnabled

public HoodieWriteConfig.Builder withEmbeddedTimelineServerReuseEnabled(boolean enabled)

withEmbeddedTimelineServerPort

public HoodieWriteConfig.Builder withEmbeddedTimelineServerPort(int port)

withBulkInsertSortMode

public HoodieWriteConfig.Builder withBulkInsertSortMode(String mode)

withAllowMultiWriteOnSameInstant

public HoodieWriteConfig.Builder withAllowMultiWriteOnSameInstant(boolean allow)

withHiveStylePartitioningEnabled

public HoodieWriteConfig.Builder withHiveStylePartitioningEnabled(boolean enabled)

withExternalSchemaTrasformation

public HoodieWriteConfig.Builder withExternalSchemaTrasformation(boolean enabled)

withMergeDataValidationCheckEnabled

public HoodieWriteConfig.Builder withMergeDataValidationCheckEnabled(boolean enabled)

withMergeAllowDuplicateOnInserts

public HoodieWriteConfig.Builder withMergeAllowDuplicateOnInserts(boolean routeInsertsToNewFiles)

withMergeSmallFileGroupCandidatesLimit

public HoodieWriteConfig.Builder withMergeSmallFileGroupCandidatesLimit(int limit)

withHeartbeatIntervalInMs

public HoodieWriteConfig.Builder withHeartbeatIntervalInMs(Integer heartbeatIntervalInMs)

withHeartbeatTolerableMisses

public HoodieWriteConfig.Builder withHeartbeatTolerableMisses(Integer heartbeatTolerableMisses)

withWriteConcurrencyMode

public HoodieWriteConfig.Builder withWriteConcurrencyMode(WriteConcurrencyMode concurrencyMode)

withPopulateMetaFields

public HoodieWriteConfig.Builder withPopulateMetaFields(boolean populateMetaFields)

withAllowOperationMetadataField

public HoodieWriteConfig.Builder withAllowOperationMetadataField(boolean allowOperationMetadataField)

withFileIdPrefixProviderClassName

public HoodieWriteConfig.Builder withFileIdPrefixProviderClassName(String fileIdPrefixProviderClassName)

withTableServicesEnabled

public HoodieWriteConfig.Builder withTableServicesEnabled(boolean enabled)

withReleaseResourceEnabled

public HoodieWriteConfig.Builder withReleaseResourceEnabled(boolean enabled)

withProperties

public HoodieWriteConfig.Builder withProperties(Properties properties)

withAutoAdjustLockConfigs

public HoodieWriteConfig.Builder withAutoAdjustLockConfigs(boolean autoAdjustLockConfigs)

doSkipDefaultPartitionValidation

public HoodieWriteConfig.Builder doSkipDefaultPartitionValidation(boolean skipDefaultPartitionValidation)

withEarlyConflictDetectionEnable

public HoodieWriteConfig.Builder withEarlyConflictDetectionEnable(boolean enable)

withAsyncConflictDetectorInitialDelayMs

public HoodieWriteConfig.Builder withAsyncConflictDetectorInitialDelayMs(long intervalMs)

withAsyncConflictDetectorPeriodMs

public HoodieWriteConfig.Builder withAsyncConflictDetectorPeriodMs(long periodMs)

withEarlyConflictDetectionCheckCommitConflict

public HoodieWriteConfig.Builder withEarlyConflictDetectionCheckCommitConflict(boolean enable)

withEarlyConflictDetectionStrategy

public HoodieWriteConfig.Builder withEarlyConflictDetectionStrategy(String className)

withRollbackBackupEnabled

public HoodieWriteConfig.Builder withRollbackBackupEnabled(boolean rollbackBackupEnabled)

withRollbackBackupDirectory

public HoodieWriteConfig.Builder withRollbackBackupDirectory(String backupDir)

withClientInitCallbackClassNames

public HoodieWriteConfig.Builder withClientInitCallbackClassNames(String classNames)

withWritesFileIdEncoding

public HoodieWriteConfig.Builder withWritesFileIdEncoding(Integer fileIdEncoding)

setDefaults

protected void setDefaults()

build

public HoodieWriteConfig build()

build

public HoodieWriteConfig build(boolean shouldValidate)