org.apache.hudi.common.engine

Class HoodieEngineContext

java.lang.Object

org.apache.hudi.common.engine.HoodieEngineContext

Direct Known Subclasses:

HoodieLocalEngineContext

public abstract class HoodieEngineContext
extends Object

Base class contains the context information needed by the engine at runtime. It will be extended by different engine implementation if needed.

Field Summary

Fields

Modifier and Type	Field and Description
protected TaskContextSupplier	taskContextSupplier

Constructor Summary

Constructors

Constructor and Description
HoodieEngineContext(SerializableConfiguration hadoopConf, TaskContextSupplier taskContextSupplier)

Method Summary

Modifier and Type	Method and Description
abstract <T> HoodieData<T>	emptyHoodieData()
abstract <I,O> List<O>	flatMap(List<I> data, SerializableFunction<I,Stream<O>> func, int parallelism)
abstract <I> void	foreach(List<I> data, SerializableConsumer<I> consumer, int parallelism)
SerializableConfiguration	getHadoopConf()
abstract Option<String>	getProperty(EngineProperty key)
TaskContextSupplier	getTaskContextSupplier()
abstract <I,O> List<O>	map(List<I> data, SerializableFunction<I,O> func, int parallelism)
abstract <I,K,V> Stream<ImmutablePair<K,V>>	mapPartitionsToPairAndReduceByKey(Stream<I> data, SerializablePairFlatMapFunction<Iterator<I>,K,V> flatMapToPairFunc, SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
abstract <I,K,V> Map<K,V>	mapToPair(List<I> data, SerializablePairFunction<I,K,V> func, Integer parallelism)
abstract <I,K,V> List<V>	mapToPairAndReduceByKey(List<I> data, SerializablePairFunction<I,K,V> mapToPairFunc, SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
abstract HoodieAccumulator	newAccumulator()
<T> HoodieData<T>	parallelize(List<T> data)
abstract <T> HoodieData<T>	parallelize(List<T> data, int parallelism)
abstract <I,K,V> List<V>	reduceByKey(List<Pair<K,V>> data, SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
abstract void	setJobStatus(String activeModule, String activityDescription)
abstract void	setProperty(EngineProperty key, String value)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

taskContextSupplier

protected TaskContextSupplier taskContextSupplier

Constructor Detail

HoodieEngineContext

public HoodieEngineContext(SerializableConfiguration hadoopConf,
                           TaskContextSupplier taskContextSupplier)

Method Detail

getHadoopConf

public SerializableConfiguration getHadoopConf()

getTaskContextSupplier

public TaskContextSupplier getTaskContextSupplier()

newAccumulator

public abstract HoodieAccumulator newAccumulator()

emptyHoodieData

public abstract <T> HoodieData<T> emptyHoodieData()

parallelize

public <T> HoodieData<T> parallelize(List<T> data)

parallelize

public abstract <T> HoodieData<T> parallelize(List<T> data,
                                              int parallelism)

map

public abstract <I,O> List<O> map(List<I> data,
                                  SerializableFunction<I,O> func,
                                  int parallelism)

mapToPairAndReduceByKey

public abstract <I,K,V> List<V> mapToPairAndReduceByKey(List<I> data,
                                                        SerializablePairFunction<I,K,V> mapToPairFunc,
                                                        SerializableBiFunction<V,V,V> reduceFunc,
                                                        int parallelism)

mapPartitionsToPairAndReduceByKey

public abstract <I,K,V> Stream<ImmutablePair<K,V>> mapPartitionsToPairAndReduceByKey(Stream<I> data,
                                                                                     SerializablePairFlatMapFunction<Iterator<I>,K,V> flatMapToPairFunc,
                                                                                     SerializableBiFunction<V,V,V> reduceFunc,
                                                                                     int parallelism)

reduceByKey

public abstract <I,K,V> List<V> reduceByKey(List<Pair<K,V>> data,
                                            SerializableBiFunction<V,V,V> reduceFunc,
                                            int parallelism)

flatMap

public abstract <I,O> List<O> flatMap(List<I> data,
                                      SerializableFunction<I,Stream<O>> func,
                                      int parallelism)

foreach

public abstract <I> void foreach(List<I> data,
                                 SerializableConsumer<I> consumer,
                                 int parallelism)

mapToPair

public abstract <I,K,V> Map<K,V> mapToPair(List<I> data,
                                           SerializablePairFunction<I,K,V> func,
                                           Integer parallelism)

setProperty

public abstract void setProperty(EngineProperty key,
                                 String value)

getProperty

public abstract Option<String> getProperty(EngineProperty key)

setJobStatus

public abstract void setJobStatus(String activeModule,
                                  String activityDescription)