org.apache.hudi.io

Class HoodieWriteHandle<T,I,K,O>

java.lang.Object

org.apache.hudi.io.HoodieIOHandle<T,I,K,O>

org.apache.hudi.io.HoodieWriteHandle<T,I,K,O>

Direct Known Subclasses:

HoodieAppendHandle, HoodieCreateHandle, HoodieMergeHandle

public abstract class HoodieWriteHandle<T,I,K,O>
extends HoodieIOHandle<T,I,K,O>

Base class for all write operations logically performed at the file group level.

Field Summary

Fields

Modifier and Type	Field and Description
protected String	fileId
protected HoodieRecordLocation	newRecordLocation
protected String	partitionPath
protected HoodieRecordMerger	recordMerger
protected boolean	schemaOnReadEnabled
protected TaskContextSupplier	taskContextSupplier
protected HoodieTimer	timer
protected org.apache.avro.Schema	writeSchema Schema used to write records into data files
protected org.apache.avro.Schema	writeSchemaWithMetaFields
protected WriteStatus	writeStatus
protected String	writeToken

Fields inherited from class org.apache.hudi.io.HoodieIOHandle

config, fs, hoodieTable, instantTime

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	HoodieWriteHandle(HoodieWriteConfig config, String instantTime, String partitionPath, String fileId, HoodieTable<T,I,K,O> hoodieTable, Option<org.apache.avro.Schema> overriddenSchema, TaskContextSupplier taskContextSupplier)
	HoodieWriteHandle(HoodieWriteConfig config, String instantTime, String partitionPath, String fileId, HoodieTable<T,I,K,O> hoodieTable, TaskContextSupplier taskContextSupplier)

Method Summary

Modifier and Type	Method and Description
boolean	canWrite(HoodieRecord record) Determines whether we can accept the incoming records, into the current file.
abstract List<WriteStatus>	close()
protected HoodieLogFormat.Writer	createLogWriter(Option<FileSlice> fileSlice, String baseCommitTime)
protected HoodieLogFormat.Writer	createLogWriter(Option<FileSlice> fileSlice, String baseCommitTime, String suffix)
protected HoodieLogFormat.Writer	createLogWriter(String baseCommitTime, String fileSuffix)
protected void	createMarkerFile(String partitionPath, String dataFileName) Creates an empty marker file corresponding to storage writer path.
protected void	doWrite(HoodieRecord record, org.apache.avro.Schema schema, TypedProperties props) Perform the actual writing of the given record into the backing file.
protected long	getAttemptId()
HoodieWriteConfig	getConfig()
String	getFileId()
org.apache.hadoop.fs.FileSystem	getFileSystem()
HoodieTableMetaClient	getHoodieTableMetaClient()
abstract IOType	getIOType()
protected int	getPartitionId()
String	getPartitionPath()
protected int	getStageId()
org.apache.avro.Schema	getWriterSchema()
org.apache.avro.Schema	getWriterSchemaWithMetaFields()
protected boolean	isClosed()
protected org.apache.hadoop.fs.Path	makeNewFilePath(String partitionPath, String fileName) Make new file path with given file name.
org.apache.hadoop.fs.Path	makeNewPath(String partitionPath)
protected void	markClosed()
protected static Option<org.apache.avro.generic.IndexedRecord>	toAvroRecord(HoodieRecord record, org.apache.avro.Schema writerSchema, TypedProperties props)
void	write(HoodieRecord record, org.apache.avro.Schema schema, TypedProperties props) Perform the actual writing of the given record into the backing file.
List<WriteStatus>	writeStatuses()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

writeSchema

protected final org.apache.avro.Schema writeSchema

Schema used to write records into data files

writeSchemaWithMetaFields

protected final org.apache.avro.Schema writeSchemaWithMetaFields

recordMerger

protected final HoodieRecordMerger recordMerger

timer

protected HoodieTimer timer

writeStatus

protected WriteStatus writeStatus

newRecordLocation

protected HoodieRecordLocation newRecordLocation

partitionPath

protected final String partitionPath

fileId

protected final String fileId

writeToken

protected final String writeToken

taskContextSupplier

protected final TaskContextSupplier taskContextSupplier

schemaOnReadEnabled

protected final boolean schemaOnReadEnabled

Constructor Detail

HoodieWriteHandle

public HoodieWriteHandle(HoodieWriteConfig config,
                         String instantTime,
                         String partitionPath,
                         String fileId,
                         HoodieTable<T,I,K,O> hoodieTable,
                         TaskContextSupplier taskContextSupplier)

HoodieWriteHandle

protected HoodieWriteHandle(HoodieWriteConfig config,
                            String instantTime,
                            String partitionPath,
                            String fileId,
                            HoodieTable<T,I,K,O> hoodieTable,
                            Option<org.apache.avro.Schema> overriddenSchema,
                            TaskContextSupplier taskContextSupplier)

Method Detail

makeNewPath

public org.apache.hadoop.fs.Path makeNewPath(String partitionPath)

makeNewFilePath

protected org.apache.hadoop.fs.Path makeNewFilePath(String partitionPath,
                                                    String fileName)

Make new file path with given file name.

createMarkerFile

protected void createMarkerFile(String partitionPath,
                                String dataFileName)

Creates an empty marker file corresponding to storage writer path.

Parameters:

partitionPath - Partition path

getWriterSchemaWithMetaFields

public org.apache.avro.Schema getWriterSchemaWithMetaFields()

getWriterSchema

public org.apache.avro.Schema getWriterSchema()

canWrite

public boolean canWrite(HoodieRecord record)

Determines whether we can accept the incoming records, into the current file. Depending on

- Whether it belongs to the same partitionPath as existing records - Whether the current file written bytes lt max file size

doWrite

protected void doWrite(HoodieRecord record,
                       org.apache.avro.Schema schema,
                       TypedProperties props)

Perform the actual writing of the given record into the backing file.

write

public void write(HoodieRecord record,
                  org.apache.avro.Schema schema,
                  TypedProperties props)

Perform the actual writing of the given record into the backing file.

isClosed

protected boolean isClosed()

markClosed

protected void markClosed()

close

public abstract List<WriteStatus> close()

writeStatuses

public List<WriteStatus> writeStatuses()

getPartitionPath

public String getPartitionPath()

getIOType

public abstract IOType getIOType()

getFileSystem

public org.apache.hadoop.fs.FileSystem getFileSystem()

Specified by:

getFileSystem in class HoodieIOHandle<T,I,K,O>

getConfig

public HoodieWriteConfig getConfig()

getHoodieTableMetaClient

public HoodieTableMetaClient getHoodieTableMetaClient()

getFileId

public String getFileId()

getPartitionId

protected int getPartitionId()

getStageId

protected int getStageId()

getAttemptId

protected long getAttemptId()

createLogWriter

protected HoodieLogFormat.Writer createLogWriter(Option<FileSlice> fileSlice,
                                                 String baseCommitTime)
                                          throws IOException

Throws:

IOException

createLogWriter

protected HoodieLogFormat.Writer createLogWriter(Option<FileSlice> fileSlice,
                                                 String baseCommitTime,
                                                 String suffix)
                                          throws IOException

Throws:

IOException

createLogWriter

protected HoodieLogFormat.Writer createLogWriter(String baseCommitTime,
                                                 String fileSuffix)

toAvroRecord

protected static Option<org.apache.avro.generic.IndexedRecord> toAvroRecord(HoodieRecord record,
                                                                            org.apache.avro.Schema writerSchema,
                                                                            TypedProperties props)