org.apache.hudi.table.action.commit

Class BaseJavaCommitActionExecutor<T>

java.lang.Object

org.apache.hudi.table.action.BaseActionExecutor<T,I,K,O,R>

org.apache.hudi.table.action.commit.BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

org.apache.hudi.table.action.commit.BaseJavaCommitActionExecutor<T>

All Implemented Interfaces:

Serializable

Direct Known Subclasses:

BaseJavaDeltaCommitActionExecutor, JavaBulkInsertCommitActionExecutor, JavaBulkInsertPreppedCommitActionExecutor, JavaDeleteCommitActionExecutor, JavaExecuteClusteringCommitActionExecutor, JavaInsertCommitActionExecutor, JavaInsertOverwriteCommitActionExecutor, JavaInsertPreppedCommitActionExecutor, JavaUpsertCommitActionExecutor, JavaUpsertPreppedCommitActionExecutor

public abstract class BaseJavaCommitActionExecutor<T>
extends BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

See Also:

Serialized Form

Field Summary

Fields inherited from class org.apache.hudi.table.action.commit.BaseCommitActionExecutor

extraMetadata, lastCompletedTxn, operationType, pendingInflightAndRequestedInstants, taskContextSupplier, txnManager

Fields inherited from class org.apache.hudi.table.action.BaseActionExecutor

config, context, hadoopConf, instantTime, table

Constructor Summary

Constructors

Constructor and Description
BaseJavaCommitActionExecutor(org.apache.hudi.common.engine.HoodieEngineContext context, HoodieWriteConfig config, HoodieTable table, String instantTime, org.apache.hudi.common.model.WriteOperationType operationType)
BaseJavaCommitActionExecutor(org.apache.hudi.common.engine.HoodieEngineContext context, HoodieWriteConfig config, HoodieTable table, String instantTime, org.apache.hudi.common.model.WriteOperationType operationType, org.apache.hudi.common.util.Option extraMetadata)

Method Summary

Modifier and Type	Method and Description
protected org.apache.hudi.common.util.collection.Pair<HashMap<String,WorkloadStat>,WorkloadStat>	buildProfile(List<org.apache.hudi.common.model.HoodieRecord<T>> inputRecords)
protected void	commit(org.apache.hudi.common.util.Option<Map<String,String>> extraMetadata, HoodieWriteMetadata<List<WriteStatus>> result)
protected void	commit(org.apache.hudi.common.util.Option<Map<String,String>> extraMetadata, HoodieWriteMetadata<List<WriteStatus>> result, List<org.apache.hudi.common.model.HoodieWriteStat> writeStats)
HoodieWriteMetadata<List<WriteStatus>>	execute(List<org.apache.hudi.common.model.HoodieRecord<T>> inputRecords)
protected String	getCommitActionType()
Partitioner	getInsertPartitioner(WorkloadProfile profile) Provides a partitioner to perform the insert operation, based on the workload profile.
protected Map<String,List<String>>	getPartitionToReplacedFileIds(HoodieWriteMetadata<List<WriteStatus>> writeMetadata)
protected HoodieMergeHandle	getUpdateHandle(String partitionPath, String fileId, Iterator<org.apache.hudi.common.model.HoodieRecord<T>> recordItr)
Partitioner	getUpsertPartitioner(WorkloadProfile profile) Provides a partitioner to perform the upsert operation, based on the workload profile.
Iterator<List<WriteStatus>>	handleInsert(String idPfx, Iterator<org.apache.hudi.common.model.HoodieRecord<T>> recordItr)
protected Iterator<List<WriteStatus>>	handleInsertPartition(String instantTime, Integer partition, Iterator recordItr, Partitioner partitioner)
Iterator<List<WriteStatus>>	handleUpdate(String partitionPath, String fileId, Iterator<org.apache.hudi.common.model.HoodieRecord<T>> recordItr)
protected Iterator<List<WriteStatus>>	handleUpdateInternal(HoodieMergeHandle<?,?,?,?> upsertHandle, String fileId)
protected Iterator<List<WriteStatus>>	handleUpsertPartition(String instantTime, Integer partition, Iterator recordItr, Partitioner partitioner)
protected boolean	isWorkloadProfileNeeded()
protected void	setCommitMetadata(HoodieWriteMetadata<List<WriteStatus>> result)
protected List<WriteStatus>	updateIndex(List<WriteStatus> writeStatuses, HoodieWriteMetadata<List<WriteStatus>> result)
void	updateIndexAndCommitIfNeeded(List<WriteStatus> writeStatuses, HoodieWriteMetadata result)

Methods inherited from class org.apache.hudi.table.action.commit.BaseCommitActionExecutor

autoCommit, commitOnAutoCommit, executeClustering, finalizeWrite, getSchemaToStoreInCommit, runPrecommitValidators

Methods inherited from class org.apache.hudi.table.action.BaseActionExecutor

execute, writeTableMetadata, writeTableMetadata, writeTableMetadata, writeTableMetadata

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

BaseJavaCommitActionExecutor

public BaseJavaCommitActionExecutor(org.apache.hudi.common.engine.HoodieEngineContext context,
                                    HoodieWriteConfig config,
                                    HoodieTable table,
                                    String instantTime,
                                    org.apache.hudi.common.model.WriteOperationType operationType)

BaseJavaCommitActionExecutor

public BaseJavaCommitActionExecutor(org.apache.hudi.common.engine.HoodieEngineContext context,
                                    HoodieWriteConfig config,
                                    HoodieTable table,
                                    String instantTime,
                                    org.apache.hudi.common.model.WriteOperationType operationType,
                                    org.apache.hudi.common.util.Option extraMetadata)

Method Detail

execute

public HoodieWriteMetadata<List<WriteStatus>> execute(List<org.apache.hudi.common.model.HoodieRecord<T>> inputRecords)

Specified by:

execute in class BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

updateIndex

protected List<WriteStatus> updateIndex(List<WriteStatus> writeStatuses,
                                        HoodieWriteMetadata<List<WriteStatus>> result)

getCommitActionType

protected String getCommitActionType()

Overrides:

getCommitActionType in class BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

buildProfile

protected org.apache.hudi.common.util.collection.Pair<HashMap<String,WorkloadStat>,WorkloadStat> buildProfile(List<org.apache.hudi.common.model.HoodieRecord<T>> inputRecords)

commit

protected void commit(org.apache.hudi.common.util.Option<Map<String,String>> extraMetadata,
                      HoodieWriteMetadata<List<WriteStatus>> result)

Specified by:

commit in class BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

setCommitMetadata

protected void setCommitMetadata(HoodieWriteMetadata<List<WriteStatus>> result)

Specified by:

setCommitMetadata in class BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

commit

protected void commit(org.apache.hudi.common.util.Option<Map<String,String>> extraMetadata,
                      HoodieWriteMetadata<List<WriteStatus>> result,
                      List<org.apache.hudi.common.model.HoodieWriteStat> writeStats)

getPartitionToReplacedFileIds

protected Map<String,List<String>> getPartitionToReplacedFileIds(HoodieWriteMetadata<List<WriteStatus>> writeMetadata)

isWorkloadProfileNeeded

protected boolean isWorkloadProfileNeeded()

Overrides:

isWorkloadProfileNeeded in class BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

handleUpsertPartition

protected Iterator<List<WriteStatus>> handleUpsertPartition(String instantTime,
                                                            Integer partition,
                                                            Iterator recordItr,
                                                            Partitioner partitioner)

handleInsertPartition

protected Iterator<List<WriteStatus>> handleInsertPartition(String instantTime,
                                                            Integer partition,
                                                            Iterator recordItr,
                                                            Partitioner partitioner)

handleUpdate

public Iterator<List<WriteStatus>> handleUpdate(String partitionPath,
                                                String fileId,
                                                Iterator<org.apache.hudi.common.model.HoodieRecord<T>> recordItr)
                                         throws IOException

Specified by:

handleUpdate in class BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

Throws:

IOException

handleUpdateInternal

protected Iterator<List<WriteStatus>> handleUpdateInternal(HoodieMergeHandle<?,?,?,?> upsertHandle,
                                                           String fileId)
                                                    throws IOException

Throws:

IOException

getUpdateHandle

protected HoodieMergeHandle getUpdateHandle(String partitionPath,
                                            String fileId,
                                            Iterator<org.apache.hudi.common.model.HoodieRecord<T>> recordItr)

handleInsert

public Iterator<List<WriteStatus>> handleInsert(String idPfx,
                                                Iterator<org.apache.hudi.common.model.HoodieRecord<T>> recordItr)

Specified by:

handleInsert in class BaseCommitActionExecutor<T,List<org.apache.hudi.common.model.HoodieRecord<T>>,List<org.apache.hudi.common.model.HoodieKey>,List<WriteStatus>,HoodieWriteMetadata>

getUpsertPartitioner

public Partitioner getUpsertPartitioner(WorkloadProfile profile)

Provides a partitioner to perform the upsert operation, based on the workload profile.

getInsertPartitioner

public Partitioner getInsertPartitioner(WorkloadProfile profile)

Provides a partitioner to perform the insert operation, based on the workload profile.

updateIndexAndCommitIfNeeded

public void updateIndexAndCommitIfNeeded(List<WriteStatus> writeStatuses,
                                         HoodieWriteMetadata result)