org.apache.hudi.integ.testsuite

Class HoodieTestSuiteWriter

java.lang.Object

org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter

All Implemented Interfaces:

Serializable

public class HoodieTestSuiteWriter
extends Object
implements Serializable

A writer abstraction for the Hudi test suite. This class wraps different implementations of writers used to perform write operations into the target hudi dataset. Current supported writers are HoodieDeltaStreamerWrapper and SparkRDDWriteClient.

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
protected HoodieTestSuiteJob.HoodieTestSuiteConfig	cfg

Constructor Summary

Constructors

Constructor and Description
HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc, Properties props, HoodieTestSuiteJob.HoodieTestSuiteConfig cfg, String schema)

Method Summary

Modifier and Type	Method and Description
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	bulkInsert(org.apache.hudi.common.util.Option<String> instantTime)
void	commit(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records, org.apache.spark.api.java.JavaRDD<DeltaWriteStats> generatedDataStats, org.apache.hudi.common.util.Option<String> instantTime)
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	compact(org.apache.hudi.common.util.Option<String> instantTime)
org.apache.hudi.common.util.collection.Pair<SchemaProvider,org.apache.hudi.common.util.collection.Pair<String,org.apache.spark.api.java.JavaRDD<org.apache.hudi.common.model.HoodieRecord>>>	fetchSource()
HoodieTestSuiteJob.HoodieTestSuiteConfig	getCfg()
org.apache.hadoop.conf.Configuration	getConfiguration()
HoodieDeltaStreamerWrapper	getDeltaStreamerWrapper()
org.apache.hudi.common.util.Option<String>	getLastCheckpoint()
org.apache.spark.rdd.RDD<org.apache.avro.generic.GenericRecord>	getNextBatch()
Properties	getProps()
String	getSchema()
org.apache.spark.api.java.JavaSparkContext	getSparkContext()
org.apache.hudi.client.SparkRDDWriteClient	getWriteClient(DagNode dagNode)
org.apache.hudi.config.HoodieWriteConfig	getWriteConfig()
void	inlineClustering()
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	insert(org.apache.hudi.common.util.Option<String> instantTime)
org.apache.hudi.common.util.Option<String>	scheduleCompaction(org.apache.hudi.common.util.Option<Map<String,String>> previousCommitExtraMetadata)
org.apache.hudi.common.util.Option<String>	startCommit()
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	upsert(org.apache.hudi.common.util.Option<String> instantTime)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

cfg

protected HoodieTestSuiteJob.HoodieTestSuiteConfig cfg

Constructor Detail

HoodieTestSuiteWriter

public HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc,
                             Properties props,
                             HoodieTestSuiteJob.HoodieTestSuiteConfig cfg,
                             String schema)
                      throws Exception

Throws:

Exception

Method Detail

getWriteConfig

public org.apache.hudi.config.HoodieWriteConfig getWriteConfig()

getNextBatch

public org.apache.spark.rdd.RDD<org.apache.avro.generic.GenericRecord> getNextBatch()
                                                                             throws Exception

Throws:

Exception

fetchSource

public org.apache.hudi.common.util.collection.Pair<SchemaProvider,org.apache.hudi.common.util.collection.Pair<String,org.apache.spark.api.java.JavaRDD<org.apache.hudi.common.model.HoodieRecord>>> fetchSource()
                                                                                                                                                                                                         throws Exception

Throws:

Exception

startCommit

public org.apache.hudi.common.util.Option<String> startCommit()

upsert

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> upsert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                             throws Exception

Throws:

Exception

insert

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> insert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                             throws Exception

Throws:

Exception

bulkInsert

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> bulkInsert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                 throws Exception

Throws:

Exception

compact

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> compact(org.apache.hudi.common.util.Option<String> instantTime)
                                                                              throws Exception

Throws:

Exception

inlineClustering

public void inlineClustering()

scheduleCompaction

public org.apache.hudi.common.util.Option<String> scheduleCompaction(org.apache.hudi.common.util.Option<Map<String,String>> previousCommitExtraMetadata)
                                                              throws Exception

Throws:

Exception

commit

public void commit(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records,
                   org.apache.spark.api.java.JavaRDD<DeltaWriteStats> generatedDataStats,
                   org.apache.hudi.common.util.Option<String> instantTime)

getWriteClient

public org.apache.hudi.client.SparkRDDWriteClient getWriteClient(DagNode dagNode)
                                                          throws IllegalAccessException

Throws:

IllegalAccessException

getDeltaStreamerWrapper

public HoodieDeltaStreamerWrapper getDeltaStreamerWrapper()

getCfg

public HoodieTestSuiteJob.HoodieTestSuiteConfig getCfg()

getConfiguration

public org.apache.hadoop.conf.Configuration getConfiguration()

getSparkContext

public org.apache.spark.api.java.JavaSparkContext getSparkContext()

getLastCheckpoint

public org.apache.hudi.common.util.Option<String> getLastCheckpoint()

getProps

public Properties getProps()

getSchema

public String getSchema()