org.apache.hadoop.hive.ql

Class Context

java.lang.Object

org.apache.hadoop.hive.ql.Context

public class Context
extends Object

Context for Semantic Analyzers. Usage: not reusable - construct a new one for each query should call clear() at end of use to remove temporary folders

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	Context.DestClausePrefix
static class	Context.Operation These ops require special handling in various places (note that Insert into Acid table is in OTHER category)
static class	Context.RewritePolicy

Field Summary

Fields

Modifier and Type	Field and Description
protected String	calcitePlan
protected String	cboInfo
protected boolean	cboSucceeded
protected String	cmd
protected ExplainConfiguration	explainConfig
static String	EXT_PREFIX
protected List<HiveLock>	hiveLocks
protected HiveTxnManager	hiveTxnManager
protected String	optimizedSql
protected int	pathid

Constructor Summary

Constructors

Modifier	Constructor and Description
	Context(org.apache.hadoop.conf.Configuration conf)
protected	Context(Context ctx)

Method Summary

Modifier and Type	Method and Description
void	addCS(String path, org.apache.hadoop.fs.ContentSummary cs)
Context.DestClausePrefix	addDeleteOfUpdateDestNamePrefix(int pos, Context.DestClausePrefix prefix)
Context.DestClausePrefix	addDestNamePrefix(int pos, Context.DestClausePrefix prefix) Will make SemanticAnalyzer.Phase1Ctx#dest in subtree rooted at 'tree' use 'prefix'.
void	addMaterializedTable(String cteName, Table table, Statistics statistics)
void	addSubContext(Context context)
void	addViewTokenRewriteStream(String viewFullyQualifiedName, org.antlr.runtime.TokenRewriteStream tokenRewriteStream)
void	checkHeartbeaterLockException()
void	clear()
void	clear(boolean deleteResultDir)
boolean	enableUnparse()
static String	generateExecutionId() Generate a unique executionId.
String	getCalcitePlan()
String	getCboInfo()
String	getCmd() Find the original query command.
org.apache.hadoop.conf.Configuration	getConf()
org.apache.hadoop.fs.ContentSummary	getCS(org.apache.hadoop.fs.Path path)
org.apache.hadoop.fs.ContentSummary	getCS(String path)
Context.DestClausePrefix	getDestNamePrefix(ASTNode curNode, QB queryBlock) The suffix is always relative to a given ASTNode.
String	getExecutionId()
int	getExecutionIndex()
ExplainConfiguration.AnalyzeState	getExplainAnalyze()
ExplainConfiguration	getExplainConfig()
boolean	getExplainLogical() Find whether the current query is a logical explain query
org.apache.hadoop.fs.Path	getExternalTmpPath(org.apache.hadoop.fs.Path path) Get a path to store tmp data destined for external Path.
org.apache.hadoop.fs.Path	getExtTmpPathRelTo(org.apache.hadoop.fs.Path path) This is similar to getExternalTmpPath() with difference being this method returns temp path within passed in uri, whereas getExternalTmpPath() ignores passed in path and returns temp path within /tmp
org.apache.hadoop.fs.Path	getFsResultCacheDirs()
Map<String,org.apache.hadoop.fs.Path>	getFsScratchDirs()
DbTxnManager.Heartbeater	getHeartbeater()
List<HiveLock>	getHiveLocks()
HiveTxnManager	getHiveTxnManager()
boolean	getIsUpdateDeleteMerge()
Map<LoadTableDesc,WriteEntity>	getLoadTableOutputMap()
org.apache.hadoop.fs.Path	getLocalScratchDir(boolean mkdir) Create a local scratch directory on demand and return it.
org.apache.hadoop.fs.Path	getLocalTmpPath() Get a tmp path on local host to store intermediate data.
org.apache.hadoop.fs.Path	getLocation()
Table	getMaterializedTable(String cteName)
Statistics	getMaterializedTableStats(TableName tableName)
String	getMoveTaskId()
org.apache.hadoop.fs.Path	getMRScratchDir() Create a map-reduce scratch directory on demand and return it.
org.apache.hadoop.fs.Path	getMRScratchDir(boolean mkDir) Create a map-reduce scratch directory on demand and return it.
org.apache.hadoop.fs.Path	getMRTmpPath() Get a path to store map-reduce intermediate data in.
org.apache.hadoop.fs.Path	getMRTmpPath(boolean mkDir)
org.apache.hadoop.fs.Path	getMRTmpPath(URI uri)
CompilationOpContext	getOpContext()
Context.Operation	getOperation()
String	getOptimizedSql()
Map<WriteEntity,List<HiveLockObj>>	getOutputLockObjects()
List<org.apache.commons.lang3.tuple.Pair<String,String>>	getParsedTables()
Map<String,org.apache.hadoop.fs.ContentSummary>	getPathToCS()
PlanMapper	getPlanMapper()
String	getReplPolicy()
org.apache.hadoop.fs.Path	getResDir()
org.apache.hadoop.fs.Path	getResFile()
AtomicInteger	getSequencer()
StatsSource	getStatsSource()
DataInput	getStream()
org.apache.hadoop.fs.Path	getTempDirForFinalJobPath(org.apache.hadoop.fs.Path path) Create a temporary directory depending of the path specified.
org.apache.hadoop.fs.Path	getTempDirForInterimJobPath(org.apache.hadoop.fs.Path path) Create a temporary directory depending of the path specified.
Table	getTempTableForLoad()
org.antlr.runtime.TokenRewriteStream	getTokenRewriteStream()
org.antlr.runtime.TokenRewriteStream	getViewTokenRewriteStream(String viewFullyQualifiedName)
WmContext	getWmContext()
boolean	isCboSucceeded()
boolean	isDeleteBranchOfUpdate(String dest)
boolean	isExplainPlan()
boolean	isExplainSkipExecution() Find whether we should execute the current query due to explain
boolean	isHDFSCleanup()
boolean	isLoadingMaterializedView()
boolean	isLocalOnlyExecutionMode() Does Hive wants to run tasks entirely on the local machine (where the query is being compiled)? Today this translates into running hadoop jobs locally
boolean	isMRTmpFileURI(String uriStr) Check if path is for intermediate data
boolean	isNeedLockMgr()
boolean	isResultCacheDir(org.apache.hadoop.fs.Path destinationPath) Check if the path is a result cache dir for this query.
boolean	isScheduledQuery()
boolean	isSkipTableMasking()
void	removeMaterializedCTEs() Remove any created directories for CTEs.
void	removeResultCacheDir() Remove any created scratch directories.
void	removeScratchDir() Remove any created scratch directories.
void	resetOpContext()
void	resetStream()
void	restoreOriginalTracker()
void	setCalcitePlan(String calcitePlan)
void	setCboInfo(String cboInfo)
void	setCboSucceeded(boolean cboSucceeded)
void	setCmd(String cmd) Set the original query command.
void	setConf(HiveConf conf)
void	setEnableUnparse(boolean enableUnparse)
void	setExecutionIndex(int executionIndex)
void	setExplainConfig(ExplainConfiguration explainConfig)
void	setExplainPlan(boolean t)
void	setFsResultCacheDirs(org.apache.hadoop.fs.Path fsResultCacheDirs)
void	setHDFSCleanup(boolean isHDFSCleanup)
void	setHeartbeater(DbTxnManager.Heartbeater heartbeater)
void	setHiveLocks(List<HiveLock> hiveLocks)
void	setHiveTxnManager(HiveTxnManager txnMgr)
void	setIsLoadingMaterializedView(boolean isLoadingMaterializedView)
void	setIsUpdateDeleteMerge(boolean isUpdate)
void	setLocation(org.apache.hadoop.fs.Path location)
void	setNeedLockMgr(boolean needLockMgr)
void	setOpContext(CompilationOpContext opContext)
void	setOperation(Context.Operation operation)
void	setOperation(Context.Operation operation, boolean splitUpdate)
void	setOptimizedSql(String newSql)
void	setOriginalTracker(String originalTracker)
void	setParsedTables(List<org.apache.commons.lang3.tuple.Pair<String,String>> parsedTables)
void	setPlanMapper(PlanMapper planMapper)
void	setReplPolicy(String replPolicy)
void	setResDir(org.apache.hadoop.fs.Path resDir)
void	setResFile(org.apache.hadoop.fs.Path resFile)
void	setScheduledQuery(boolean scheduledQuery)
void	setSkipTableMasking(boolean skipTableMasking)
void	setStatsSource(StatsSource statsSource)
void	setTempTableForLoad(Table tempTableForLoad)
void	setTokenRewriteStream(org.antlr.runtime.TokenRewriteStream tokenRewriteStream) Set the token rewrite stream being used to parse the current top-level SQL statement.
void	setWmContext(WmContext wmContext)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

pathid

protected int pathid

explainConfig

protected ExplainConfiguration explainConfig

cboInfo

protected String cboInfo

cboSucceeded

protected boolean cboSucceeded

optimizedSql

protected String optimizedSql

calcitePlan

protected String calcitePlan

cmd

protected String cmd

hiveLocks

protected List<HiveLock> hiveLocks

hiveTxnManager

protected HiveTxnManager hiveTxnManager

EXT_PREFIX

public static final String EXT_PREFIX

See Also:

Constant Field Values

Constructor Detail

Context

public Context(org.apache.hadoop.conf.Configuration conf)

Context

protected Context(Context ctx)

Method Detail

setOperation

public void setOperation(Context.Operation operation)

setOperation

public void setOperation(Context.Operation operation,
                         boolean splitUpdate)

getOperation

public Context.Operation getOperation()

getWmContext

public WmContext getWmContext()

setWmContext

public void setWmContext(WmContext wmContext)

setReplPolicy

public void setReplPolicy(String replPolicy)

getReplPolicy

public String getReplPolicy()

getLocation

public org.apache.hadoop.fs.Path getLocation()

setLocation

public void setLocation(org.apache.hadoop.fs.Path location)

getDestNamePrefix

public Context.DestClausePrefix getDestNamePrefix(ASTNode curNode,
                                                  QB queryBlock)

The suffix is always relative to a given ASTNode. We need this so that FileSinkOperatorS corresponding to different branches of a multi-insert statement which represents a SQL Merge statement get marked correctly with AcidUtils.Operation. See usages of getDestNamePrefix(ASTNode, QB) and SemanticAnalyzer.updating(String) and SemanticAnalyzer.deleting(String).

addDestNamePrefix

public Context.DestClausePrefix addDestNamePrefix(int pos,
                                                  Context.DestClausePrefix prefix)

Will make SemanticAnalyzer.Phase1Ctx#dest in subtree rooted at 'tree' use 'prefix'. This to handle multi-insert stmt that represents Merge stmt and has insert branches representing update/delete/insert.

Parameters:

pos - ordinal index of specific TOK_INSERT as child of TOK_QUERY

Returns:

previous prefix for 'tree' or null

addDeleteOfUpdateDestNamePrefix

public Context.DestClausePrefix addDeleteOfUpdateDestNamePrefix(int pos,
                                                                Context.DestClausePrefix prefix)

getFsScratchDirs

public Map<String,org.apache.hadoop.fs.Path> getFsScratchDirs()

setFsResultCacheDirs

public void setFsResultCacheDirs(org.apache.hadoop.fs.Path fsResultCacheDirs)

getFsResultCacheDirs

public org.apache.hadoop.fs.Path getFsResultCacheDirs()

getLoadTableOutputMap

public Map<LoadTableDesc,WriteEntity> getLoadTableOutputMap()

getOutputLockObjects

public Map<WriteEntity,List<HiveLockObj>> getOutputLockObjects()

isExplainSkipExecution

public boolean isExplainSkipExecution()

Find whether we should execute the current query due to explain

Returns:

true if the query skips execution, false if does execute

getExplainLogical

public boolean getExplainLogical()

Find whether the current query is a logical explain query

getExplainAnalyze

public ExplainConfiguration.AnalyzeState getExplainAnalyze()

setCmd

public void setCmd(String cmd)

Set the original query command.

Parameters:

cmd - the original query command string

getCmd

public String getCmd()

Find the original query command.

Returns:

the original query command string

getLocalScratchDir

public org.apache.hadoop.fs.Path getLocalScratchDir(boolean mkdir)

Create a local scratch directory on demand and return it.

getMRScratchDir

public org.apache.hadoop.fs.Path getMRScratchDir(boolean mkDir)

Create a map-reduce scratch directory on demand and return it.

Parameters:

mkDir - flag to indicate if scratch dir is to be created or not

getMRScratchDir

public org.apache.hadoop.fs.Path getMRScratchDir()

Create a map-reduce scratch directory on demand and return it.

getTempDirForInterimJobPath

public org.apache.hadoop.fs.Path getTempDirForInterimJobPath(org.apache.hadoop.fs.Path path)

Create a temporary directory depending of the path specified. - If path is an Object store filesystem, then use the default MR scratch directory (HDFS), unless isFinalJob and BlobStorageUtils.areOptimizationsEnabled(Configuration) are both true, then return a path on the blobstore. - If path is on HDFS, then create a staging directory inside the path

Parameters:

path - Path used to verify the Filesystem to use for temporary directory

Returns:

A path to the new temporary directory

getTempDirForFinalJobPath

public org.apache.hadoop.fs.Path getTempDirForFinalJobPath(org.apache.hadoop.fs.Path path)

Create a temporary directory depending of the path specified. - If path is an Object store filesystem, then use the default MR scratch directory (HDFS) - If path is on HDFS, then create a staging directory inside the path

Parameters:

path - Path used to verify the Filesystem to use for temporary directory

Returns:

A path to the new temporary directory

removeResultCacheDir

public void removeResultCacheDir()

Remove any created scratch directories.

removeScratchDir

public void removeScratchDir()

Remove any created scratch directories.

removeMaterializedCTEs

public void removeMaterializedCTEs()

Remove any created directories for CTEs.

isMRTmpFileURI

public boolean isMRTmpFileURI(String uriStr)

Check if path is for intermediate data

Returns:

true if a uri is a temporary uri for map-reduce intermediate data, false otherwise

isResultCacheDir

public boolean isResultCacheDir(org.apache.hadoop.fs.Path destinationPath)

Check if the path is a result cache dir for this query. This doesn't work unless the result paths have already been set in SemanticAnalyzer::getDestinationFilePath to prevent someone from overriding LOCATION in a create table command to overwrite cached results

Parameters:

destinationPath -

Returns:

true if the path is a result cache dir

getMRTmpPath

public org.apache.hadoop.fs.Path getMRTmpPath(URI uri)

getMRTmpPath

public org.apache.hadoop.fs.Path getMRTmpPath(boolean mkDir)

getMRTmpPath

public org.apache.hadoop.fs.Path getMRTmpPath()

Get a path to store map-reduce intermediate data in.

Returns:

next available path for map-red intermediate data

getLocalTmpPath

public org.apache.hadoop.fs.Path getLocalTmpPath()

Get a tmp path on local host to store intermediate data.

Returns:

next available tmp path on local fs

getExternalTmpPath

public org.apache.hadoop.fs.Path getExternalTmpPath(org.apache.hadoop.fs.Path path)

Get a path to store tmp data destined for external Path.

Parameters:

path - external Path to which the tmp data has to be eventually moved

Returns:

next available tmp path on the file system corresponding extURI

getExtTmpPathRelTo

public org.apache.hadoop.fs.Path getExtTmpPathRelTo(org.apache.hadoop.fs.Path path)

This is similar to getExternalTmpPath() with difference being this method returns temp path within passed in uri, whereas getExternalTmpPath() ignores passed in path and returns temp path within /tmp

getMoveTaskId

public String getMoveTaskId()

getResFile

public org.apache.hadoop.fs.Path getResFile()

Returns:

the resFile

setResFile

public void setResFile(org.apache.hadoop.fs.Path resFile)

Parameters:

resFile - the resFile to set

getResDir

public org.apache.hadoop.fs.Path getResDir()

Returns:

the resDir

setResDir

public void setResDir(org.apache.hadoop.fs.Path resDir)

Parameters:

resDir - the resDir to set

clear

public void clear()
           throws IOException

Throws:

IOException

clear

public void clear(boolean deleteResultDir)
           throws IOException

Throws:

IOException

getStream

public DataInput getStream()

resetStream

public void resetStream()

setTokenRewriteStream

public void setTokenRewriteStream(org.antlr.runtime.TokenRewriteStream tokenRewriteStream)

Set the token rewrite stream being used to parse the current top-level SQL statement. Note that this should not be used for other parsing activities; for example, when we encounter a reference to a view, we switch to a new stream for parsing the stored view definition from the catalog, but we don't clobber the top-level stream in the context.

Parameters:

tokenRewriteStream - the stream being used

getTokenRewriteStream

public org.antlr.runtime.TokenRewriteStream getTokenRewriteStream()

Returns:

the token rewrite stream being used to parse the current top-level SQL statement, or null if it isn't available (e.g. for parser tests)

addViewTokenRewriteStream

public void addViewTokenRewriteStream(String viewFullyQualifiedName,
                                      org.antlr.runtime.TokenRewriteStream tokenRewriteStream)

getViewTokenRewriteStream

public org.antlr.runtime.TokenRewriteStream getViewTokenRewriteStream(String viewFullyQualifiedName)

generateExecutionId

public static String generateExecutionId()

Generate a unique executionId. An executionId, together with user name and the configuration, will determine the temporary locations of all intermediate files. In the future, users can use the executionId to resume a query.

isLocalOnlyExecutionMode

public boolean isLocalOnlyExecutionMode()

Does Hive wants to run tasks entirely on the local machine (where the query is being compiled)? Today this translates into running hadoop jobs locally

getHiveLocks

public List<HiveLock> getHiveLocks()

setHiveLocks

public void setHiveLocks(List<HiveLock> hiveLocks)

getHiveTxnManager

public HiveTxnManager getHiveTxnManager()

setHiveTxnManager

public void setHiveTxnManager(HiveTxnManager txnMgr)

setOriginalTracker

public void setOriginalTracker(String originalTracker)

restoreOriginalTracker

public void restoreOriginalTracker()

addSubContext

public void addSubContext(Context context)

addCS

public void addCS(String path,
                  org.apache.hadoop.fs.ContentSummary cs)

getCS

public org.apache.hadoop.fs.ContentSummary getCS(org.apache.hadoop.fs.Path path)

getCS

public org.apache.hadoop.fs.ContentSummary getCS(String path)

getPathToCS

public Map<String,org.apache.hadoop.fs.ContentSummary> getPathToCS()

getConf

public org.apache.hadoop.conf.Configuration getConf()

isHDFSCleanup

public boolean isHDFSCleanup()

Returns:

the isHDFSCleanup

setHDFSCleanup

public void setHDFSCleanup(boolean isHDFSCleanup)

Parameters:

isHDFSCleanup - the isHDFSCleanup to set

isNeedLockMgr

public boolean isNeedLockMgr()

setNeedLockMgr

public void setNeedLockMgr(boolean needLockMgr)

getCboInfo

public String getCboInfo()

setCboInfo

public void setCboInfo(String cboInfo)

getOptimizedSql

public String getOptimizedSql()

setOptimizedSql

public void setOptimizedSql(String newSql)

isCboSucceeded

public boolean isCboSucceeded()

setCboSucceeded

public void setCboSucceeded(boolean cboSucceeded)

getCalcitePlan

public String getCalcitePlan()

setCalcitePlan

public void setCalcitePlan(String calcitePlan)

getMaterializedTable

public Table getMaterializedTable(String cteName)

addMaterializedTable

public void addMaterializedTable(String cteName,
                                 Table table,
                                 Statistics statistics)

getMaterializedTableStats

public Statistics getMaterializedTableStats(TableName tableName)

getSequencer

public AtomicInteger getSequencer()

getOpContext

public CompilationOpContext getOpContext()

setOpContext

public void setOpContext(CompilationOpContext opContext)

getHeartbeater

public DbTxnManager.Heartbeater getHeartbeater()

setHeartbeater

public void setHeartbeater(DbTxnManager.Heartbeater heartbeater)

checkHeartbeaterLockException

public void checkHeartbeaterLockException()
                                   throws LockException

Throws:

LockException

isSkipTableMasking

public boolean isSkipTableMasking()

setSkipTableMasking

public void setSkipTableMasking(boolean skipTableMasking)

getExplainConfig

public ExplainConfiguration getExplainConfig()

isExplainPlan

public boolean isExplainPlan()

setExplainPlan

public void setExplainPlan(boolean t)

setExplainConfig

public void setExplainConfig(ExplainConfiguration explainConfig)

resetOpContext

public void resetOpContext()

getIsUpdateDeleteMerge

public boolean getIsUpdateDeleteMerge()

setIsUpdateDeleteMerge

public void setIsUpdateDeleteMerge(boolean isUpdate)

isLoadingMaterializedView

public boolean isLoadingMaterializedView()

setIsLoadingMaterializedView

public void setIsLoadingMaterializedView(boolean isLoadingMaterializedView)

getExecutionId

public String getExecutionId()

setPlanMapper

public void setPlanMapper(PlanMapper planMapper)

getPlanMapper

public PlanMapper getPlanMapper()

setStatsSource

public void setStatsSource(StatsSource statsSource)

getStatsSource

public StatsSource getStatsSource()

getExecutionIndex

public int getExecutionIndex()

setExecutionIndex

public void setExecutionIndex(int executionIndex)

setConf

public void setConf(HiveConf conf)

getTempTableForLoad

public Table getTempTableForLoad()

setTempTableForLoad

public void setTempTableForLoad(Table tempTableForLoad)

enableUnparse

public boolean enableUnparse()

setEnableUnparse

public void setEnableUnparse(boolean enableUnparse)

isScheduledQuery

public boolean isScheduledQuery()

setScheduledQuery

public void setScheduledQuery(boolean scheduledQuery)

setParsedTables

public void setParsedTables(List<org.apache.commons.lang3.tuple.Pair<String,String>> parsedTables)

getParsedTables

public List<org.apache.commons.lang3.tuple.Pair<String,String>> getParsedTables()

isDeleteBranchOfUpdate

public boolean isDeleteBranchOfUpdate(String dest)