org.apache.hadoop.hive.ql.plan

Class FileSinkDesc

java.lang.Object

org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

org.apache.hadoop.hive.ql.plan.FileSinkDesc

All Implemented Interfaces:

Serializable, Cloneable, IStatsGatherDesc, OperatorDesc

public class FileSinkDesc
extends AbstractOperatorDesc
implements IStatsGatherDesc

FileSinkDesc.

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	FileSinkDesc.DPSortState
class	FileSinkDesc.FileSinkOperatorExplainVectorization

Field Summary

Fields inherited from class org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

bucketingVersion, colExprMap, memAvailable, memNeeded, opProps, opTraits, runtimeStatsTmpDir, statistics, vectorDesc, vectorMode

Constructor Summary

Constructors

Constructor and Description
FileSinkDesc()
FileSinkDesc(org.apache.hadoop.fs.Path dirName, TableDesc tableInfo, boolean compressed)
FileSinkDesc(org.apache.hadoop.fs.Path dirName, TableDesc tableInfo, boolean compressed, int destTableId, boolean multiFileSpray, boolean canBeMerged, int numFiles, int totalFiles, List<ExprNodeDesc> partitionCols, DynamicPartitionCtx dpCtx, org.apache.hadoop.fs.Path destPath, Long mmWriteId, boolean isMmCtas, boolean isInsertOverwrite, boolean isQuery, boolean isCTASorCM, boolean isDirectInsert, AcidUtils.Operation acidOperation, boolean deleteOfSplitUpdate)

Method Summary

Modifier and Type	Method and Description
boolean	canBeMerged()
Object	clone()
AcidUtils.Operation	getAcidOperation()
int	getBucketingVersionForExplain()
String	getCompressCodec()
boolean	getCompressed()
String	getCompressType()
org.apache.hadoop.fs.Path	getDestPath()
int	getDestTableId()
org.apache.hadoop.fs.Path	getDirName()
String	getDirNameString()
FileSinkDesc.DPSortState	getDpSortState()
String	getDpSortStateString()
DynamicPartitionCtx	getDynPartCtx()
Map<String,List<org.apache.hadoop.fs.Path>>	getDynPartitionValues()
FileSinkDesc.FileSinkOperatorExplainVectorization	getFileSinkVectorization()
Set<org.apache.hadoop.fs.FileStatus>	getFilesToFetch()
org.apache.hadoop.fs.Path	getFinalDirName()
boolean	getInsertOverwrite()
boolean	getIsQuery()
ListBucketingCtx	getLbCtx()
List<FileSinkDesc>	getLinkedFileSinkDesc()
int	getMaxStmtId()
org.apache.hadoop.fs.Path	getMergeInputDirName() getFinalDirName that takes into account MM, but not DP, LB or buckets.
String	getMoveTaskId()
int	getNumFiles()
org.apache.hadoop.fs.Path	getParentDir()
List<ExprNodeDesc>	getPartitionCols()
int	getStatementId() See AcidOutputFormat.Options.statementId(int)
String	getStaticSpec()
String	getStatsAggPrefix() Construct the key prefix used as (intermediate) statistics publishing and aggregation.
Table	getTable()
TableDesc	getTableInfo()
long	getTableWriteId()
String	getTmpStatsDir()
int	getTotalFiles()
Context.Operation	getWriteOperation()
AcidUtils.Operation	getWriteType()
String	getWriteTypeString()
boolean	isCompactionTable()
boolean	isCTASorCM() Whether this is CREATE TABLE SELECT or CREATE MATERIALIZED VIEW statement Set by semantic analyzer this is required because CTAS/CM requires some special logic in mvFileToFinalPath
boolean	isDeleteOfSplitUpdate()
boolean	isDirectInsert()
boolean	isFullAcidTable()
boolean	isGatherStats()
boolean	isHiveServerQuery()
boolean	isLinkedFileSink()
boolean	isMaterialization()
boolean	isMerge()
boolean	isMmCtas()
boolean	isMmTable()
boolean	isMultiFileSpray()
boolean	isRebalanceRequested()
boolean	isRemovedReduceSinkBucketSort()
boolean	isSame(OperatorDesc other) The default implementation delegates to Object.equals(Object).
boolean	isStatsReliable()
boolean	isTemporary()
boolean	isUsingBatchingSerDe()
void	setAcidOperation(AcidUtils.Operation acidOperation)
void	setCanBeMerged(boolean canBeMerged)
void	setCompressCodec(String intermediateCompressorCodec)
void	setCompressed(boolean compressed)
void	setCompressType(String intermediateCompressType)
void	setDestTableId(int destTableId)
void	setDirName(org.apache.hadoop.fs.Path dirName)
void	setDpSortState(FileSinkDesc.DPSortState dpSortState)
void	setDynPartCtx(DynamicPartitionCtx dpc)
void	setDynPartitionValues(Map<String,List<org.apache.hadoop.fs.Path>> dynPartitionValues)
void	setFilesToFetch(Set<org.apache.hadoop.fs.FileStatus> filesToFetch)
void	setGatherStats(boolean gatherStats)
void	setHiveServerQuery(boolean isHiveServerQuery)
void	setInsertOverwrite(boolean isInsertOverwrite)
void	setIsCTASorCM(boolean isCTASorCM)
void	setIsDirectInsert(boolean isDirectInsert)
void	setIsMerge(boolean b)
void	setIsQuery(boolean isQuery)
void	setIsUsingBatchingSerDe(boolean isUsingBatchingSerDe)
void	setLbCtx(ListBucketingCtx lbCtx)
void	setLinkedFileSink(boolean linkedFileSink)
void	setLinkedFileSinkDesc(List<FileSinkDesc> linkedFileSinkDesc)
void	setMaterialization(boolean materialization)
void	setMaxStmtId(int maxStmtId)
void	setMmWriteId(Long mmWriteId)
void	setMoveTaskId(String moveTaskId)
void	setMultiFileSpray(boolean multiFileSpray)
void	setNumFiles(int numFiles)
void	setPartitionCols(List<ExprNodeDesc> partitionCols)
void	setRemovedReduceSinkBucketSort(boolean removedReduceSinkBucketSort)
void	setStatementId(int id)
void	setStaticSpec(String staticSpec)
void	setStatsAggPrefix(String k) Set the stats aggregation key.
void	setStatsReliable(boolean statsReliable)
void	setStatsTmpDir(String statsCollectionTempDir)
void	setTable(Table table)
void	setTableInfo(TableDesc tableInfo)
void	setTableWriteId(long id)
void	setTemporary(boolean temporary)
void	setTotalFiles(int totalFiles)
void	setWriteOperation(Context.Operation writeOperation)
void	setWriteType(AcidUtils.Operation type)

Methods inherited from class org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

addComputedField, fillSignature, getBucketingVersion, getColumnExprMap, getColumnExprMapForExplain, getComputedFields, getMaxMemoryAvailable, getMemoryNeeded, getOpProps, getRuntimeStatsTmpDir, getStatistics, getTraits, getUserLevelStatistics, getVectorDesc, getVectorMode, setBucketingVersion, setColumnExprMap, setMaxMemoryAvailable, setMemoryNeeded, setOpProps, setRuntimeStatsTmpDir, setStatistics, setTraits, setVectorDesc, setVectorMode

Methods inherited from class java.lang.Object

equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

FileSinkDesc

public FileSinkDesc()

FileSinkDesc

public FileSinkDesc(org.apache.hadoop.fs.Path dirName,
                    TableDesc tableInfo,
                    boolean compressed,
                    int destTableId,
                    boolean multiFileSpray,
                    boolean canBeMerged,
                    int numFiles,
                    int totalFiles,
                    List<ExprNodeDesc> partitionCols,
                    DynamicPartitionCtx dpCtx,
                    org.apache.hadoop.fs.Path destPath,
                    Long mmWriteId,
                    boolean isMmCtas,
                    boolean isInsertOverwrite,
                    boolean isQuery,
                    boolean isCTASorCM,
                    boolean isDirectInsert,
                    AcidUtils.Operation acidOperation,
                    boolean deleteOfSplitUpdate)

Parameters:

destPath - - the final destination for data

FileSinkDesc

public FileSinkDesc(org.apache.hadoop.fs.Path dirName,
                    TableDesc tableInfo,
                    boolean compressed)

Method Detail

clone

public Object clone()
             throws CloneNotSupportedException

Specified by:

clone in interface OperatorDesc

Overrides:

clone in class AbstractOperatorDesc

Throws:

CloneNotSupportedException

setFilesToFetch

public void setFilesToFetch(Set<org.apache.hadoop.fs.FileStatus> filesToFetch)

setIsCTASorCM

public void setIsCTASorCM(boolean isCTASorCM)

setIsQuery

public void setIsQuery(boolean isQuery)

getIsQuery

public boolean getIsQuery()

getFilesToFetch

public Set<org.apache.hadoop.fs.FileStatus> getFilesToFetch()

isHiveServerQuery

public boolean isHiveServerQuery()

setHiveServerQuery

public void setHiveServerQuery(boolean isHiveServerQuery)

isUsingBatchingSerDe

public boolean isUsingBatchingSerDe()

setIsUsingBatchingSerDe

public void setIsUsingBatchingSerDe(boolean isUsingBatchingSerDe)

setIsDirectInsert

public void setIsDirectInsert(boolean isDirectInsert)

isDirectInsert

public boolean isDirectInsert()

setAcidOperation

public void setAcidOperation(AcidUtils.Operation acidOperation)

getAcidOperation

public AcidUtils.Operation getAcidOperation()

isDeleteOfSplitUpdate

public boolean isDeleteOfSplitUpdate()

getDirName

public org.apache.hadoop.fs.Path getDirName()

getDirNameString

public String getDirNameString()

setDirName

public void setDirName(org.apache.hadoop.fs.Path dirName)

getFinalDirName

public org.apache.hadoop.fs.Path getFinalDirName()

getMergeInputDirName

public org.apache.hadoop.fs.Path getMergeInputDirName()

getFinalDirName that takes into account MM, but not DP, LB or buckets.

getTableInfo

public TableDesc getTableInfo()

setTableInfo

public void setTableInfo(TableDesc tableInfo)

getCompressed

public boolean getCompressed()

setCompressed

public void setCompressed(boolean compressed)

getDestTableId

public int getDestTableId()

setDestTableId

public void setDestTableId(int destTableId)

getCompressCodec

public String getCompressCodec()

setCompressCodec

public void setCompressCodec(String intermediateCompressorCodec)

getCompressType

public String getCompressType()

setCompressType

public void setCompressType(String intermediateCompressType)

isMultiFileSpray

public boolean isMultiFileSpray()

Returns:

the multiFileSpray

setMultiFileSpray

public void setMultiFileSpray(boolean multiFileSpray)

Parameters:

multiFileSpray - the multiFileSpray to set

isTemporary

public boolean isTemporary()

Returns:

destination is temporary

setTemporary

public void setTemporary(boolean temporary)

isMmTable

public boolean isMmTable()

isFullAcidTable

public boolean isFullAcidTable()

isCompactionTable

public boolean isCompactionTable()

Returns:

true, if the table is used during compaction

isRebalanceRequested

public boolean isRebalanceRequested()

Returns:

true if the compaction type is 'REBALANCE', false otherwise.

isMaterialization

public boolean isMaterialization()

setMaterialization

public void setMaterialization(boolean materialization)

canBeMerged

public boolean canBeMerged()

setCanBeMerged

public void setCanBeMerged(boolean canBeMerged)

getTotalFiles

public int getTotalFiles()

Returns:

the totalFiles

setTotalFiles

public void setTotalFiles(int totalFiles)

Parameters:

totalFiles - the totalFiles to set

getPartitionCols

public List<ExprNodeDesc> getPartitionCols()

Returns:

the partitionCols

setPartitionCols

public void setPartitionCols(List<ExprNodeDesc> partitionCols)

Parameters:

partitionCols - the partitionCols to set

getNumFiles

public int getNumFiles()

Returns:

the numFiles

setNumFiles

public void setNumFiles(int numFiles)

Parameters:

numFiles - the numFiles to set

setDynPartCtx

public void setDynPartCtx(DynamicPartitionCtx dpc)

getDynPartCtx

public DynamicPartitionCtx getDynPartCtx()

setStaticSpec

public void setStaticSpec(String staticSpec)

getStaticSpec

public String getStaticSpec()

setGatherStats

public void setGatherStats(boolean gatherStats)

isGatherStats

public boolean isGatherStats()

Specified by:

isGatherStats in interface IStatsGatherDesc

getStatsAggPrefix

public String getStatsAggPrefix()

Construct the key prefix used as (intermediate) statistics publishing and aggregation. During stats publishing phase, this key prefix will be appended with the optional dynamic partition spec and the task ID. The whole key uniquely identifies the output of a task for this job. In the stats aggregation phase, all rows with the same prefix plus dynamic partition specs (obtained at run-time after MR job finishes) will be serving as the prefix: all rows with the same prefix (output of all tasks for this job) will be aggregated.

Specified by:

getStatsAggPrefix in interface IStatsGatherDesc

Returns:

key prefix used for stats publishing and aggregation.

setStatsAggPrefix

public void setStatsAggPrefix(String k)

Set the stats aggregation key. If the input string is not terminated by Path.SEPARATOR aggregation key will add one to make it as a directory name.

Parameters:

k - input directory name.

isLinkedFileSink

public boolean isLinkedFileSink()

setLinkedFileSink

public void setLinkedFileSink(boolean linkedFileSink)

getParentDir

public org.apache.hadoop.fs.Path getParentDir()

isStatsReliable

public boolean isStatsReliable()

setStatsReliable

public void setStatsReliable(boolean statsReliable)

getLbCtx

public ListBucketingCtx getLbCtx()

Returns:

the lbCtx

setLbCtx

public void setLbCtx(ListBucketingCtx lbCtx)

Parameters:

lbCtx - the lbCtx to set

getLinkedFileSinkDesc

public List<FileSinkDesc> getLinkedFileSinkDesc()

setLinkedFileSinkDesc

public void setLinkedFileSinkDesc(List<FileSinkDesc> linkedFileSinkDesc)

isRemovedReduceSinkBucketSort

public boolean isRemovedReduceSinkBucketSort()

setRemovedReduceSinkBucketSort

public void setRemovedReduceSinkBucketSort(boolean removedReduceSinkBucketSort)

getDpSortState

public FileSinkDesc.DPSortState getDpSortState()

getDpSortStateString

public String getDpSortStateString()

setDpSortState

public void setDpSortState(FileSinkDesc.DPSortState dpSortState)

setWriteType

public void setWriteType(AcidUtils.Operation type)

getWriteType

public AcidUtils.Operation getWriteType()

setWriteOperation

public void setWriteOperation(Context.Operation writeOperation)

getWriteOperation

public Context.Operation getWriteOperation()

getWriteTypeString

public String getWriteTypeString()

setTableWriteId

public void setTableWriteId(long id)

getTableWriteId

public long getTableWriteId()

setStatementId

public void setStatementId(int id)

getStatementId

public int getStatementId()

See AcidOutputFormat.Options.statementId(int)

setMaxStmtId

public void setMaxStmtId(int maxStmtId)

getMaxStmtId

public int getMaxStmtId()

getDestPath

public org.apache.hadoop.fs.Path getDestPath()

getTable

public Table getTable()

setTable

public void setTable(Table table)

getTmpStatsDir

public String getTmpStatsDir()

Specified by:

getTmpStatsDir in interface IStatsGatherDesc

setStatsTmpDir

public void setStatsTmpDir(String statsCollectionTempDir)

setMmWriteId

public void setMmWriteId(Long mmWriteId)

setIsMerge

public void setIsMerge(boolean b)

isMerge

public boolean isMerge()

isMmCtas

public boolean isMmCtas()

getBucketingVersionForExplain

public int getBucketingVersionForExplain()

isCTASorCM

public boolean isCTASorCM()

Whether this is CREATE TABLE SELECT or CREATE MATERIALIZED VIEW statement Set by semantic analyzer this is required because CTAS/CM requires some special logic in mvFileToFinalPath

getFileSinkVectorization

public FileSinkDesc.FileSinkOperatorExplainVectorization getFileSinkVectorization()

setInsertOverwrite

public void setInsertOverwrite(boolean isInsertOverwrite)

getInsertOverwrite

public boolean getInsertOverwrite()

isSame

public boolean isSame(OperatorDesc other)

Description copied from class: AbstractOperatorDesc

The default implementation delegates to Object.equals(Object). Intended to be overridden by sub classes.

Specified by:

isSame in interface OperatorDesc

Overrides:

isSame in class AbstractOperatorDesc

getMoveTaskId

public String getMoveTaskId()

setMoveTaskId

public void setMoveTaskId(String moveTaskId)

getDynPartitionValues

public Map<String,List<org.apache.hadoop.fs.Path>> getDynPartitionValues()

setDynPartitionValues

public void setDynPartitionValues(Map<String,List<org.apache.hadoop.fs.Path>> dynPartitionValues)