org.apache.hadoop.hive.ql.exec.repl.util

Class ReplUtils

java.lang.Object

org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils

public class ReplUtils
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	ReplUtils.MetricName Replication Metrics.
static class	ReplUtils.ReplLoadOpType Bootstrap REPL LOAD operation type on the examined object based on ckpt state.
static class	ReplUtils.TimeSerializer

Field Summary

Fields

Modifier and Type	Field and Description
static String	CONSTRAINTS_ROOT_DIR_NAME
static String	DEFAULT_FS_CONFIG
static String	DFS_MAX_DIR_ITEMS_CONFIG
static String	DISTCP_JOB_ID_CONF
static String	DISTCP_JOB_ID_CONF_DEFAULT
static String	FUNCTIONS_ROOT_DIR_NAME
static String	HIVE_RANGER_POLICIES_FILE_NAME
static String	INC_BOOTSTRAP_ROOT_DIR_NAME
static String	LAST_REPL_ID_KEY
static String	RANGER_AUTHORIZER
static String	RANGER_CONFIGURATION_RESOURCE_NAME
static String	RANGER_HIVE_SERVICE_NAME
static String	RANGER_REST_URL
static String	REPL_ATLAS_BASE_DIR
static String	REPL_ATLAS_EXPORT_FILE_NAME
static String	REPL_ATLAS_SERVICE
static String	REPL_DUMP_INCLUDE_ACID_TABLES
static String	REPL_HIVE_BASE_DIR
static String	REPL_HIVE_SERVICE
static String	REPL_RANGER_BASE_DIR
static String	REPL_RANGER_SERVICE
static String	REPL_TABLE_LIST_DIR_NAME
static int	RESERVED_DIR_ITEMS_COUNT

Constructor Summary

Constructors

Constructor and Description
ReplUtils()

Method Summary

Modifier and Type	Method and Description
static List<Task<?>>	addChildTask(Task<?> childTask)
static void	addLoggerTask(ReplLogger replLogger, List<Task<?>> tasks, HiveConf conf) Adds a logger task at the end of the tasks passed.
static List<Task<?>>	addTasksForLoadingColStats(org.apache.hadoop.hive.metastore.api.ColumnStatistics colStats, HiveConf conf, UpdatedMetaDataTracker updatedMetadata, org.apache.hadoop.hive.metastore.api.Table tableObj, long writeId, String nonRecoverableMarkPath, ReplicationMetricCollector metricCollector)
static String	convertToHumanReadableTime(long time) Convert to a human time of minutes:seconds.millis.
static boolean	failedWithNonRecoverableError(org.apache.hadoop.fs.Path dumpRoot, HiveConf conf)
static boolean	filterTransactionOperations(HiveConf conf)
static Map<Integer,List<ExprNodeGenericFuncDesc>>	genPartSpecs(Table table, List<Map<String,String>> partitions)
static org.apache.hadoop.fs.PathFilter	getBootstrapDirectoryFilter(org.apache.hadoop.fs.FileSystem fs)
static String	getDistCpCustomName(HiveConf conf, String dbName)
static org.apache.hadoop.fs.Path	getEncodedDumpRootPath(HiveConf conf, String dbname)
static MessageDeserializer	getEventDeserializer(org.apache.hadoop.hive.metastore.api.NotificationEvent event)
static org.apache.hadoop.fs.PathFilter	getEventsDirectoryFilter(org.apache.hadoop.fs.FileSystem fs)
static org.apache.hadoop.fs.Path	getLatestDumpPath(org.apache.hadoop.fs.Path dumpRoot, HiveConf conf)
static String	getNonEmpty(String configParam, HiveConf hiveConf, String errorMsgFormat)
static List<Long>	getOpenTxns(HiveTxnManager hiveTxnManager, ValidTxnList validTxnList, String dbName)
static List<Long>	getOpenTxns(ValidTxnList validTxnList)
static Task<?>	getTableCheckpointTask(ImportTableDesc tableDesc, HashMap<String,String> partSpec, String dumpRoot, ReplicationMetricCollector metricCollector, HiveConf conf)
static Task<?>	getTableReplLogTask(ImportTableDesc tableDesc, ReplLogger replLogger, HiveConf conf, ReplicationMetricCollector metricCollector, String dumpRoot)
static int	handleException(boolean isReplication, Throwable e, String nonRecoverablePath, ReplicationMetricCollector metricCollector, String stageName, HiveConf conf)
static boolean	includeAcidTableInDump(HiveConf conf)
static boolean	isErrorRecoverable(Throwable e)
static boolean	isFirstIncPending(Map<String,String> parameters)
static boolean	replCkptStatus(String dbName, Map<String,String> props, String dumpRoot)
static void	reportStatusInReplicationMetrics(String stageName, Status status, String errorLogPath, HiveConf conf)
static org.apache.hadoop.hive.metastore.api.EnvironmentContext	setReplDataLocationChangedFlag(org.apache.hadoop.hive.metastore.api.EnvironmentContext envContext)
static boolean	tableIncludedInReplScope(org.apache.hadoop.hive.common.repl.ReplScope replScope, String tableName)
static void	unsetDbPropIfSet(org.apache.hadoop.hive.metastore.api.Database db, String prop, Hive hiveDb)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LAST_REPL_ID_KEY

public static final String LAST_REPL_ID_KEY

See Also:

Constant Field Values

FUNCTIONS_ROOT_DIR_NAME

public static final String FUNCTIONS_ROOT_DIR_NAME

See Also:

Constant Field Values

CONSTRAINTS_ROOT_DIR_NAME

public static final String CONSTRAINTS_ROOT_DIR_NAME

See Also:

Constant Field Values

INC_BOOTSTRAP_ROOT_DIR_NAME

public static final String INC_BOOTSTRAP_ROOT_DIR_NAME

See Also:

Constant Field Values

REPL_HIVE_BASE_DIR

public static final String REPL_HIVE_BASE_DIR

See Also:

Constant Field Values

REPL_RANGER_BASE_DIR

public static final String REPL_RANGER_BASE_DIR

See Also:

Constant Field Values

REPL_ATLAS_BASE_DIR

public static final String REPL_ATLAS_BASE_DIR

See Also:

Constant Field Values

REPL_ATLAS_EXPORT_FILE_NAME

public static final String REPL_ATLAS_EXPORT_FILE_NAME

See Also:

Constant Field Values

DEFAULT_FS_CONFIG

public static final String DEFAULT_FS_CONFIG

See Also:

Constant Field Values

REPL_TABLE_LIST_DIR_NAME

public static final String REPL_TABLE_LIST_DIR_NAME

See Also:

Constant Field Values

REPL_DUMP_INCLUDE_ACID_TABLES

public static final String REPL_DUMP_INCLUDE_ACID_TABLES

See Also:

Constant Field Values

DFS_MAX_DIR_ITEMS_CONFIG

public static final String DFS_MAX_DIR_ITEMS_CONFIG

See Also:

Constant Field Values

RESERVED_DIR_ITEMS_COUNT

public static final int RESERVED_DIR_ITEMS_COUNT

See Also:

Constant Field Values

RANGER_AUTHORIZER

public static final String RANGER_AUTHORIZER

See Also:

Constant Field Values

HIVE_RANGER_POLICIES_FILE_NAME

public static final String HIVE_RANGER_POLICIES_FILE_NAME

See Also:

Constant Field Values

RANGER_REST_URL

public static final String RANGER_REST_URL

See Also:

Constant Field Values

RANGER_HIVE_SERVICE_NAME

public static final String RANGER_HIVE_SERVICE_NAME

See Also:

Constant Field Values

RANGER_CONFIGURATION_RESOURCE_NAME

public static final String RANGER_CONFIGURATION_RESOURCE_NAME

See Also:

Constant Field Values

REPL_HIVE_SERVICE

public static final String REPL_HIVE_SERVICE

See Also:

Constant Field Values

REPL_RANGER_SERVICE

public static final String REPL_RANGER_SERVICE

See Also:

Constant Field Values

REPL_ATLAS_SERVICE

public static final String REPL_ATLAS_SERVICE

See Also:

Constant Field Values

DISTCP_JOB_ID_CONF

public static final String DISTCP_JOB_ID_CONF

See Also:

Constant Field Values

DISTCP_JOB_ID_CONF_DEFAULT

public static final String DISTCP_JOB_ID_CONF_DEFAULT

See Also:

Constant Field Values

Constructor Detail

ReplUtils

public ReplUtils()

Method Detail

genPartSpecs

public static Map<Integer,List<ExprNodeGenericFuncDesc>> genPartSpecs(Table table,
                                                                      List<Map<String,String>> partitions)
                                                               throws SemanticException

Throws:

SemanticException

unsetDbPropIfSet

public static void unsetDbPropIfSet(org.apache.hadoop.hive.metastore.api.Database db,
                                    String prop,
                                    Hive hiveDb)
                             throws HiveException

Throws:

HiveException

getTableReplLogTask

public static Task<?> getTableReplLogTask(ImportTableDesc tableDesc,
                                          ReplLogger replLogger,
                                          HiveConf conf,
                                          ReplicationMetricCollector metricCollector,
                                          String dumpRoot)
                                   throws SemanticException

Throws:

SemanticException

getTableCheckpointTask

public static Task<?> getTableCheckpointTask(ImportTableDesc tableDesc,
                                             HashMap<String,String> partSpec,
                                             String dumpRoot,
                                             ReplicationMetricCollector metricCollector,
                                             HiveConf conf)
                                      throws SemanticException

Throws:

SemanticException

replCkptStatus

public static boolean replCkptStatus(String dbName,
                                     Map<String,String> props,
                                     String dumpRoot)
                              throws org.apache.hadoop.hive.metastore.api.InvalidOperationException

Throws:

org.apache.hadoop.hive.metastore.api.InvalidOperationException

getNonEmpty

public static String getNonEmpty(String configParam,
                                 HiveConf hiveConf,
                                 String errorMsgFormat)
                          throws SemanticException

Throws:

SemanticException

addChildTask

public static List<Task<?>> addChildTask(Task<?> childTask)

addTasksForLoadingColStats

public static List<Task<?>> addTasksForLoadingColStats(org.apache.hadoop.hive.metastore.api.ColumnStatistics colStats,
                                                       HiveConf conf,
                                                       UpdatedMetaDataTracker updatedMetadata,
                                                       org.apache.hadoop.hive.metastore.api.Table tableObj,
                                                       long writeId,
                                                       String nonRecoverableMarkPath,
                                                       ReplicationMetricCollector metricCollector)
                                                throws IOException,
                                                       org.apache.thrift.TException

Throws:

IOException

org.apache.thrift.TException

getEventsDirectoryFilter

public static org.apache.hadoop.fs.PathFilter getEventsDirectoryFilter(org.apache.hadoop.fs.FileSystem fs)

getBootstrapDirectoryFilter

public static org.apache.hadoop.fs.PathFilter getBootstrapDirectoryFilter(org.apache.hadoop.fs.FileSystem fs)

handleException

public static int handleException(boolean isReplication,
                                  Throwable e,
                                  String nonRecoverablePath,
                                  ReplicationMetricCollector metricCollector,
                                  String stageName,
                                  HiveConf conf)

isFirstIncPending

public static boolean isFirstIncPending(Map<String,String> parameters)

getOpenTxns

public static List<Long> getOpenTxns(ValidTxnList validTxnList)

getOpenTxns

public static List<Long> getOpenTxns(HiveTxnManager hiveTxnManager,
                                     ValidTxnList validTxnList,
                                     String dbName)
                              throws LockException

Throws:

LockException

getEventDeserializer

public static MessageDeserializer getEventDeserializer(org.apache.hadoop.hive.metastore.api.NotificationEvent event)

setReplDataLocationChangedFlag

public static org.apache.hadoop.hive.metastore.api.EnvironmentContext setReplDataLocationChangedFlag(org.apache.hadoop.hive.metastore.api.EnvironmentContext envContext)

includeAcidTableInDump

public static boolean includeAcidTableInDump(HiveConf conf)

tableIncludedInReplScope

public static boolean tableIncludedInReplScope(org.apache.hadoop.hive.common.repl.ReplScope replScope,
                                               String tableName)

failedWithNonRecoverableError

public static boolean failedWithNonRecoverableError(org.apache.hadoop.fs.Path dumpRoot,
                                                    HiveConf conf)
                                             throws SemanticException

Throws:

SemanticException

getEncodedDumpRootPath

public static org.apache.hadoop.fs.Path getEncodedDumpRootPath(HiveConf conf,
                                                               String dbname)
                                                        throws UnsupportedEncodingException

Throws:

UnsupportedEncodingException

getLatestDumpPath

public static org.apache.hadoop.fs.Path getLatestDumpPath(org.apache.hadoop.fs.Path dumpRoot,
                                                          HiveConf conf)
                                                   throws IOException

Throws:

IOException

getDistCpCustomName

public static String getDistCpCustomName(HiveConf conf,
                                         String dbName)

convertToHumanReadableTime

public static String convertToHumanReadableTime(long time)

Convert to a human time of minutes:seconds.millis.

Parameters:

time - time to humanize.

Returns:

a printable value.

addLoggerTask

public static void addLoggerTask(ReplLogger replLogger,
                                 List<Task<?>> tasks,
                                 HiveConf conf)

Adds a logger task at the end of the tasks passed.

reportStatusInReplicationMetrics

public static void reportStatusInReplicationMetrics(String stageName,
                                                    Status status,
                                                    String errorLogPath,
                                                    HiveConf conf)
                                             throws SemanticException

Throws:

SemanticException

isErrorRecoverable

public static boolean isErrorRecoverable(Throwable e)

filterTransactionOperations

public static boolean filterTransactionOperations(HiveConf conf)