org.apache.hudi.io

Class HoodieMergeHandle<T,I,K,O>

java.lang.Object

org.apache.hudi.io.HoodieIOHandle<T,I,K,O>

org.apache.hudi.io.HoodieWriteHandle<T,I,K,O>

org.apache.hudi.io.HoodieMergeHandle<T,I,K,O>

Direct Known Subclasses:

HoodieConcatHandle, HoodieMergeHandleWithChangeLog, HoodieSortedMergeHandle

@NotThreadSafe
public class HoodieMergeHandle<T,I,K,O>
extends HoodieWriteHandle<T,I,K,O>

Field Summary

Fields

Modifier and Type	Field and Description
protected HoodieFileWriter	fileWriter
protected long	insertRecordsWritten
protected Option<BaseKeyGenerator>	keyGeneratorOpt
protected Map<String,HoodieRecord<T>>	keyToNewRecords
protected org.apache.hadoop.fs.Path	newFilePath
protected org.apache.hadoop.fs.Path	oldFilePath
protected Option<String[]>	partitionFields
protected Object[]	partitionValues
protected long	recordsDeleted
protected long	recordsWritten
protected long	updatedRecordsWritten
protected boolean	useWriterSchemaForCompaction
protected Set<String>	writtenRecordKeys

Fields inherited from class org.apache.hudi.io.HoodieWriteHandle

fileId, newRecordLocation, partitionPath, recordMerger, schemaOnReadEnabled, taskContextSupplier, timer, writeSchema, writeSchemaWithMetaFields, writeStatus, writeToken

Fields inherited from class org.apache.hudi.io.HoodieIOHandle

config, fs, hoodieTable, instantTime

Constructor Summary

Constructors

Constructor and Description
HoodieMergeHandle(HoodieWriteConfig config, String instantTime, HoodieTable<T,I,K,O> hoodieTable, Iterator<HoodieRecord<T>> recordItr, String partitionPath, String fileId, TaskContextSupplier taskContextSupplier, HoodieBaseFile baseFile, Option<BaseKeyGenerator> keyGeneratorOpt)
HoodieMergeHandle(HoodieWriteConfig config, String instantTime, HoodieTable<T,I,K,O> hoodieTable, Iterator<HoodieRecord<T>> recordItr, String partitionPath, String fileId, TaskContextSupplier taskContextSupplier, Option<BaseKeyGenerator> keyGeneratorOpt)
HoodieMergeHandle(HoodieWriteConfig config, String instantTime, HoodieTable<T,I,K,O> hoodieTable, Map<String,HoodieRecord<T>> keyToNewRecords, String partitionPath, String fileId, HoodieBaseFile dataFileToBeMerged, TaskContextSupplier taskContextSupplier, Option<BaseKeyGenerator> keyGeneratorOpt) Called by compactor code path.

Method Summary

Modifier and Type	Method and Description
HoodieBaseFile	baseFileForMerge()
List<WriteStatus>	close()
IOType	getIOType()
static HoodieBaseFile	getLatestBaseFile(HoodieTable<?,?,?,?> hoodieTable, String partitionPath, String fileId)
org.apache.hadoop.fs.Path	getOldFilePath()
Option<String[]>	getPartitionFields()
Object[]	getPartitionValues()
protected void	init(String fileId, Iterator<HoodieRecord<T>> newRecordsItr) Load the new incoming records in a map and return partitionPath.
protected void	initializeIncomingRecordsMap() Initialize a spillable map for incoming records.
protected void	makeOldAndNewFilePaths(String partitionPath, String oldFileName, String newFileName)
void	performMergeDataValidationCheck(WriteStatus writeStatus)
void	setPartitionFields(Option<String[]> partitionFields)
void	setPartitionValues(Object[] partitionValues)
protected void	setWriteStatusPath()
void	write(HoodieRecord<T> oldRecord) Go through an old record.
protected void	writeIncomingRecords()
protected void	writeInsertRecord(HoodieRecord<T> newRecord)
protected void	writeInsertRecord(HoodieRecord<T> newRecord, org.apache.avro.Schema schema, Properties prop)
protected boolean	writeRecord(HoodieRecord<T> newRecord, Option<HoodieRecord> combineRecord, org.apache.avro.Schema schema, Properties prop)
protected void	writeToFile(HoodieKey key, HoodieRecord<T> record, org.apache.avro.Schema schema, Properties prop, boolean shouldPreserveRecordMetadata)
protected boolean	writeUpdateRecord(HoodieRecord<T> newRecord, HoodieRecord<T> oldRecord, Option<HoodieRecord> combineRecordOpt, org.apache.avro.Schema writerSchema)

Methods inherited from class org.apache.hudi.io.HoodieWriteHandle

canWrite, createLogWriter, createLogWriter, createLogWriter, createMarkerFile, doWrite, getAttemptId, getConfig, getFileId, getFileSystem, getHoodieTableMetaClient, getPartitionId, getPartitionPath, getStageId, getWriterSchema, getWriterSchemaWithMetaFields, isClosed, makeNewFilePath, makeNewPath, markClosed, toAvroRecord, write, writeStatuses

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

keyToNewRecords

protected Map<String,HoodieRecord<T>> keyToNewRecords

writtenRecordKeys

protected Set<String> writtenRecordKeys

fileWriter

protected HoodieFileWriter fileWriter

newFilePath

protected org.apache.hadoop.fs.Path newFilePath

oldFilePath

protected org.apache.hadoop.fs.Path oldFilePath

recordsWritten

protected long recordsWritten

recordsDeleted

protected long recordsDeleted

updatedRecordsWritten

protected long updatedRecordsWritten

insertRecordsWritten

protected long insertRecordsWritten

useWriterSchemaForCompaction

protected boolean useWriterSchemaForCompaction

keyGeneratorOpt

protected Option<BaseKeyGenerator> keyGeneratorOpt

partitionFields

protected Option<String[]> partitionFields

partitionValues

protected Object[] partitionValues

Constructor Detail

HoodieMergeHandle

public HoodieMergeHandle(HoodieWriteConfig config,
                         String instantTime,
                         HoodieTable<T,I,K,O> hoodieTable,
                         Iterator<HoodieRecord<T>> recordItr,
                         String partitionPath,
                         String fileId,
                         TaskContextSupplier taskContextSupplier,
                         Option<BaseKeyGenerator> keyGeneratorOpt)

HoodieMergeHandle

public HoodieMergeHandle(HoodieWriteConfig config,
                         String instantTime,
                         HoodieTable<T,I,K,O> hoodieTable,
                         Iterator<HoodieRecord<T>> recordItr,
                         String partitionPath,
                         String fileId,
                         TaskContextSupplier taskContextSupplier,
                         HoodieBaseFile baseFile,
                         Option<BaseKeyGenerator> keyGeneratorOpt)

HoodieMergeHandle

public HoodieMergeHandle(HoodieWriteConfig config,
                         String instantTime,
                         HoodieTable<T,I,K,O> hoodieTable,
                         Map<String,HoodieRecord<T>> keyToNewRecords,
                         String partitionPath,
                         String fileId,
                         HoodieBaseFile dataFileToBeMerged,
                         TaskContextSupplier taskContextSupplier,
                         Option<BaseKeyGenerator> keyGeneratorOpt)

Called by compactor code path.

Method Detail

getLatestBaseFile

public static HoodieBaseFile getLatestBaseFile(HoodieTable<?,?,?,?> hoodieTable,
                                               String partitionPath,
                                               String fileId)

setWriteStatusPath

protected void setWriteStatusPath()

makeOldAndNewFilePaths

protected void makeOldAndNewFilePaths(String partitionPath,
                                      String oldFileName,
                                      String newFileName)

initializeIncomingRecordsMap

protected void initializeIncomingRecordsMap()

Initialize a spillable map for incoming records.

init

protected void init(String fileId,
                    Iterator<HoodieRecord<T>> newRecordsItr)

Load the new incoming records in a map and return partitionPath.

writeUpdateRecord

protected boolean writeUpdateRecord(HoodieRecord<T> newRecord,
                                    HoodieRecord<T> oldRecord,
                                    Option<HoodieRecord> combineRecordOpt,
                                    org.apache.avro.Schema writerSchema)
                             throws IOException

Throws:

IOException

writeInsertRecord

protected void writeInsertRecord(HoodieRecord<T> newRecord)
                          throws IOException

Throws:

IOException

writeInsertRecord

protected void writeInsertRecord(HoodieRecord<T> newRecord,
                                 org.apache.avro.Schema schema,
                                 Properties prop)
                          throws IOException

Throws:

IOException

writeRecord

protected boolean writeRecord(HoodieRecord<T> newRecord,
                              Option<HoodieRecord> combineRecord,
                              org.apache.avro.Schema schema,
                              Properties prop)
                       throws IOException

Throws:

IOException

write

public void write(HoodieRecord<T> oldRecord)

Go through an old record. Here if we detect a newer version shows up, we write the new one to the file.

writeToFile

protected void writeToFile(HoodieKey key,
                           HoodieRecord<T> record,
                           org.apache.avro.Schema schema,
                           Properties prop,
                           boolean shouldPreserveRecordMetadata)
                    throws IOException

Throws:

IOException

writeIncomingRecords

protected void writeIncomingRecords()
                             throws IOException

Throws:

IOException

close

public List<WriteStatus> close()

Specified by:

close in class HoodieWriteHandle<T,I,K,O>

performMergeDataValidationCheck

public void performMergeDataValidationCheck(WriteStatus writeStatus)

getOldFilePath

public org.apache.hadoop.fs.Path getOldFilePath()

getIOType

public IOType getIOType()

Specified by:

getIOType in class HoodieWriteHandle<T,I,K,O>

baseFileForMerge

public HoodieBaseFile baseFileForMerge()

setPartitionFields

public void setPartitionFields(Option<String[]> partitionFields)

getPartitionFields

public Option<String[]> getPartitionFields()

setPartitionValues

public void setPartitionValues(Object[] partitionValues)

getPartitionValues

public Object[] getPartitionValues()