org.apache.hudi.client.common

Class HoodieJavaEngineContext

java.lang.Object

org.apache.hudi.common.engine.HoodieEngineContext

org.apache.hudi.client.common.HoodieJavaEngineContext

public class HoodieJavaEngineContext
extends org.apache.hudi.common.engine.HoodieEngineContext

A java engine implementation of HoodieEngineContext.

Field Summary

Fields inherited from class org.apache.hudi.common.engine.HoodieEngineContext

taskContextSupplier

Constructor Summary

Constructors

Constructor and Description
HoodieJavaEngineContext(org.apache.hadoop.conf.Configuration conf)
HoodieJavaEngineContext(org.apache.hadoop.conf.Configuration conf, org.apache.hudi.common.engine.TaskContextSupplier taskContextSupplier)

Method Summary

Modifier and Type	Method and Description
<T> org.apache.hudi.common.data.HoodieData<T>	emptyHoodieData()
<I,O> List<O>	flatMap(List<I> data, org.apache.hudi.common.function.SerializableFunction<I,Stream<O>> func, int parallelism)
<I> void	foreach(List<I> data, org.apache.hudi.common.function.SerializableConsumer<I> consumer, int parallelism)
org.apache.hudi.common.util.Option<String>	getProperty(org.apache.hudi.common.engine.EngineProperty key)
<I,O> List<O>	map(List<I> data, org.apache.hudi.common.function.SerializableFunction<I,O> func, int parallelism)
<I,K,V> Stream<org.apache.hudi.common.util.collection.ImmutablePair<K,V>>	mapPartitionsToPairAndReduceByKey(Stream<I> data, org.apache.hudi.common.function.SerializablePairFlatMapFunction<Iterator<I>,K,V> flatMapToPairFunc, org.apache.hudi.common.function.SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
<I,K,V> Map<K,V>	mapToPair(List<I> data, org.apache.hudi.common.function.SerializablePairFunction<I,K,V> func, Integer parallelism)
<I,K,V> List<V>	mapToPairAndReduceByKey(List<I> data, org.apache.hudi.common.function.SerializablePairFunction<I,K,V> mapToPairFunc, org.apache.hudi.common.function.SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
org.apache.hudi.common.data.HoodieAccumulator	newAccumulator()
<T> org.apache.hudi.common.data.HoodieData<T>	parallelize(List<T> data, int parallelism)
<I,K,V> List<V>	reduceByKey(List<org.apache.hudi.common.util.collection.Pair<K,V>> data, org.apache.hudi.common.function.SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
void	setJobStatus(String activeModule, String activityDescription)
void	setProperty(org.apache.hudi.common.engine.EngineProperty key, String value)

Methods inherited from class org.apache.hudi.common.engine.HoodieEngineContext

getHadoopConf, getTaskContextSupplier, parallelize

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HoodieJavaEngineContext

public HoodieJavaEngineContext(org.apache.hadoop.conf.Configuration conf)

HoodieJavaEngineContext

public HoodieJavaEngineContext(org.apache.hadoop.conf.Configuration conf,
                               org.apache.hudi.common.engine.TaskContextSupplier taskContextSupplier)

Method Detail

newAccumulator

public org.apache.hudi.common.data.HoodieAccumulator newAccumulator()

Specified by:

newAccumulator in class org.apache.hudi.common.engine.HoodieEngineContext

emptyHoodieData

public <T> org.apache.hudi.common.data.HoodieData<T> emptyHoodieData()

Specified by:

emptyHoodieData in class org.apache.hudi.common.engine.HoodieEngineContext

parallelize

public <T> org.apache.hudi.common.data.HoodieData<T> parallelize(List<T> data,
                                                                 int parallelism)

Specified by:

parallelize in class org.apache.hudi.common.engine.HoodieEngineContext

map

public <I,O> List<O> map(List<I> data,
                         org.apache.hudi.common.function.SerializableFunction<I,O> func,
                         int parallelism)

Specified by:

map in class org.apache.hudi.common.engine.HoodieEngineContext

mapToPairAndReduceByKey

public <I,K,V> List<V> mapToPairAndReduceByKey(List<I> data,
                                               org.apache.hudi.common.function.SerializablePairFunction<I,K,V> mapToPairFunc,
                                               org.apache.hudi.common.function.SerializableBiFunction<V,V,V> reduceFunc,
                                               int parallelism)

Specified by:

mapToPairAndReduceByKey in class org.apache.hudi.common.engine.HoodieEngineContext

mapPartitionsToPairAndReduceByKey

public <I,K,V> Stream<org.apache.hudi.common.util.collection.ImmutablePair<K,V>> mapPartitionsToPairAndReduceByKey(Stream<I> data,
                                                                                                                   org.apache.hudi.common.function.SerializablePairFlatMapFunction<Iterator<I>,K,V> flatMapToPairFunc,
                                                                                                                   org.apache.hudi.common.function.SerializableBiFunction<V,V,V> reduceFunc,
                                                                                                                   int parallelism)

Specified by:

mapPartitionsToPairAndReduceByKey in class org.apache.hudi.common.engine.HoodieEngineContext

reduceByKey

public <I,K,V> List<V> reduceByKey(List<org.apache.hudi.common.util.collection.Pair<K,V>> data,
                                   org.apache.hudi.common.function.SerializableBiFunction<V,V,V> reduceFunc,
                                   int parallelism)

Specified by:

reduceByKey in class org.apache.hudi.common.engine.HoodieEngineContext

flatMap

public <I,O> List<O> flatMap(List<I> data,
                             org.apache.hudi.common.function.SerializableFunction<I,Stream<O>> func,
                             int parallelism)

Specified by:

flatMap in class org.apache.hudi.common.engine.HoodieEngineContext

foreach

public <I> void foreach(List<I> data,
                        org.apache.hudi.common.function.SerializableConsumer<I> consumer,
                        int parallelism)

Specified by:

foreach in class org.apache.hudi.common.engine.HoodieEngineContext

mapToPair

public <I,K,V> Map<K,V> mapToPair(List<I> data,
                                  org.apache.hudi.common.function.SerializablePairFunction<I,K,V> func,
                                  Integer parallelism)

Specified by:

mapToPair in class org.apache.hudi.common.engine.HoodieEngineContext

setProperty

public void setProperty(org.apache.hudi.common.engine.EngineProperty key,
                        String value)

Specified by:

setProperty in class org.apache.hudi.common.engine.HoodieEngineContext

getProperty

public org.apache.hudi.common.util.Option<String> getProperty(org.apache.hudi.common.engine.EngineProperty key)

Specified by:

getProperty in class org.apache.hudi.common.engine.HoodieEngineContext

setJobStatus

public void setJobStatus(String activeModule,
                         String activityDescription)

Specified by:

setJobStatus in class org.apache.hudi.common.engine.HoodieEngineContext