org.apache.hudi.hadoop.utils

Class HoodieHiveUtils

java.lang.Object

org.apache.hudi.hadoop.utils.HoodieHiveUtils

public class HoodieHiveUtils
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
static int	DEFAULT_MAX_COMMITS
static String	DEFAULT_SCAN_MODE
static String	GLOBALLY_CONSISTENT_READ_TIMESTAMP
static String	HOODIE_CONSUME_COMMIT
static String	HOODIE_CONSUME_MODE_PATTERN
static Pattern	HOODIE_CONSUME_MODE_PATTERN_STRING
static String	HOODIE_CONSUME_PENDING_COMMITS
static String	HOODIE_INCREMENTAL_USE_DATABASE
static String	HOODIE_MAX_COMMIT_PATTERN
static String	HOODIE_START_COMMIT_PATTERN
static String	HOODIE_STOP_AT_COMPACTION_PATTERN
static String	INCREMENTAL_SCAN_MODE
static org.apache.log4j.Logger	LOG
static int	MAX_COMMIT_ALL
static String	SNAPSHOT_SCAN_MODE
static Set<String>	VIRTUAL_COLUMN_NAMES

Constructor Summary

Constructors

Constructor and Description
HoodieHiveUtils()

Method Summary

Modifier and Type	Method and Description
static List<String>	getIncrementalTableNames(org.apache.hadoop.mapreduce.JobContext job) Returns a list of tableNames for which hoodie..consume.mode is set to incremental else returns empty List
static Option<String>	getMaxCommit(org.apache.hadoop.mapred.JobConf job, String tableName)
static org.apache.hadoop.fs.Path	getNthParent(org.apache.hadoop.fs.Path path, int n) Gets the n'th parent for the Path.
static boolean	isIncrementalUseDatabase(org.apache.hadoop.conf.Configuration conf)
static Integer	readMaxCommits(org.apache.hadoop.mapreduce.JobContext job, String tableName)
static String	readStartCommitTime(org.apache.hadoop.mapreduce.JobContext job, String tableName)
static boolean	shouldIncludePendingCommits(org.apache.hadoop.mapred.JobConf job, String tableName)
static boolean	stopAtCompaction(org.apache.hadoop.mapreduce.JobContext job, String tableName)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

public static final org.apache.log4j.Logger LOG

HOODIE_INCREMENTAL_USE_DATABASE

public static final String HOODIE_INCREMENTAL_USE_DATABASE

See Also:

Constant Field Values

HOODIE_CONSUME_MODE_PATTERN

public static final String HOODIE_CONSUME_MODE_PATTERN

See Also:

Constant Field Values

HOODIE_START_COMMIT_PATTERN

public static final String HOODIE_START_COMMIT_PATTERN

See Also:

Constant Field Values

HOODIE_MAX_COMMIT_PATTERN

public static final String HOODIE_MAX_COMMIT_PATTERN

See Also:

Constant Field Values

HOODIE_CONSUME_PENDING_COMMITS

public static final String HOODIE_CONSUME_PENDING_COMMITS

See Also:

Constant Field Values

HOODIE_CONSUME_COMMIT

public static final String HOODIE_CONSUME_COMMIT

See Also:

Constant Field Values

VIRTUAL_COLUMN_NAMES

public static final Set<String> VIRTUAL_COLUMN_NAMES

HOODIE_STOP_AT_COMPACTION_PATTERN

public static final String HOODIE_STOP_AT_COMPACTION_PATTERN

See Also:

Constant Field Values

INCREMENTAL_SCAN_MODE

public static final String INCREMENTAL_SCAN_MODE

See Also:

Constant Field Values

SNAPSHOT_SCAN_MODE

public static final String SNAPSHOT_SCAN_MODE

See Also:

Constant Field Values

DEFAULT_SCAN_MODE

public static final String DEFAULT_SCAN_MODE

See Also:

Constant Field Values

DEFAULT_MAX_COMMITS

public static final int DEFAULT_MAX_COMMITS

See Also:

Constant Field Values

MAX_COMMIT_ALL

public static final int MAX_COMMIT_ALL

See Also:

Constant Field Values

HOODIE_CONSUME_MODE_PATTERN_STRING

public static final Pattern HOODIE_CONSUME_MODE_PATTERN_STRING

GLOBALLY_CONSISTENT_READ_TIMESTAMP

public static final String GLOBALLY_CONSISTENT_READ_TIMESTAMP

See Also:

Constant Field Values

Constructor Detail

HoodieHiveUtils

public HoodieHiveUtils()

Method Detail

shouldIncludePendingCommits

public static boolean shouldIncludePendingCommits(org.apache.hadoop.mapred.JobConf job,
                                                  String tableName)

getMaxCommit

public static Option<String> getMaxCommit(org.apache.hadoop.mapred.JobConf job,
                                          String tableName)

stopAtCompaction

public static boolean stopAtCompaction(org.apache.hadoop.mapreduce.JobContext job,
                                       String tableName)

readMaxCommits

public static Integer readMaxCommits(org.apache.hadoop.mapreduce.JobContext job,
                                     String tableName)

readStartCommitTime

public static String readStartCommitTime(org.apache.hadoop.mapreduce.JobContext job,
                                         String tableName)

getNthParent

public static org.apache.hadoop.fs.Path getNthParent(org.apache.hadoop.fs.Path path,
                                                     int n)

Gets the n'th parent for the Path. Assumes the path has at-least n components

Parameters:

path -

n -

Returns:

getIncrementalTableNames

public static List<String> getIncrementalTableNames(org.apache.hadoop.mapreduce.JobContext job)

Returns a list of tableNames for which hoodie..consume.mode is set to incremental else returns empty List

Parameters:

job -

Returns:

isIncrementalUseDatabase

public static boolean isIncrementalUseDatabase(org.apache.hadoop.conf.Configuration conf)