org.apache.hudi.integ.testsuite

Class HoodieTestSuiteWriter

java.lang.Object

org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter

All Implemented Interfaces:

Serializable

Direct Known Subclasses:

HoodieContinousTestSuiteWriter, HoodieInlineTestSuiteWriter

public abstract class HoodieTestSuiteWriter
extends Object
implements Serializable

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
protected HoodieTestSuiteJob.HoodieTestSuiteConfig	cfg
protected org.apache.hadoop.conf.Configuration	configuration
protected HoodieDeltaStreamerWrapper	deltaStreamerWrapper
protected org.apache.hudi.client.HoodieReadClient	hoodieReadClient
protected org.apache.hudi.common.util.Option<String>	lastCheckpoint
protected Properties	props
protected String	schema
protected org.apache.spark.api.java.JavaSparkContext	sparkContext
protected static Set<String>	VALID_DAG_NODES_TO_ALLOW_WRITE_CLIENT_IN_DELTASTREAMER_MODE
protected org.apache.hudi.client.SparkRDDWriteClient	writeClient
protected org.apache.hudi.config.HoodieWriteConfig	writeConfig

Constructor Summary

Constructors

Constructor and Description
HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc, Properties props, HoodieTestSuiteJob.HoodieTestSuiteConfig cfg, String schema)

Method Summary

Modifier and Type	Method and Description
abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	bulkInsert(org.apache.hudi.common.util.Option<String> instantTime)
abstract void	commit(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records, org.apache.spark.api.java.JavaRDD<DeltaWriteStats> generatedDataStats, org.apache.hudi.common.util.Option<String> instantTime)
abstract void	commitCompaction(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records, org.apache.spark.api.java.JavaRDD<DeltaWriteStats> generatedDataStats, org.apache.hudi.common.util.Option<String> instantTime)
abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	compact(org.apache.hudi.common.util.Option<String> instantTime)
abstract org.apache.hudi.common.util.collection.Pair<SchemaProvider,org.apache.hudi.common.util.collection.Pair<String,org.apache.spark.api.java.JavaRDD<org.apache.hudi.common.model.HoodieRecord>>>	fetchSource()
HoodieTestSuiteJob.HoodieTestSuiteConfig	getCfg()
org.apache.hadoop.conf.Configuration	getConfiguration()
HoodieDeltaStreamerWrapper	getDeltaStreamerWrapper()
org.apache.hudi.common.util.Option<String>	getLastCheckpoint()
abstract org.apache.spark.rdd.RDD<org.apache.avro.generic.GenericRecord>	getNextBatch()
Properties	getProps()
String	getSchema()
org.apache.spark.api.java.JavaSparkContext	getSparkContext()
org.apache.hudi.client.SparkRDDWriteClient	getWriteClient(DagNode dagNode)
org.apache.hudi.config.HoodieWriteConfig	getWriteConfig()
abstract void	inlineClustering()
abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	insert(org.apache.hudi.common.util.Option<String> instantTime)
abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	insertOverwrite(org.apache.hudi.common.util.Option<String> instantTime)
abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	insertOverwriteTable(org.apache.hudi.common.util.Option<String> instantTime)
abstract org.apache.hudi.common.util.Option<String>	scheduleCompaction(org.apache.hudi.common.util.Option<Map<String,String>> previousCommitExtraMetadata)
abstract void	shutdownResources()
abstract org.apache.hudi.common.util.Option<String>	startCommit()
abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	upsert(org.apache.hudi.common.util.Option<String> instantTime)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

deltaStreamerWrapper

protected HoodieDeltaStreamerWrapper deltaStreamerWrapper

writeConfig

protected org.apache.hudi.config.HoodieWriteConfig writeConfig

writeClient

protected org.apache.hudi.client.SparkRDDWriteClient writeClient

cfg

protected HoodieTestSuiteJob.HoodieTestSuiteConfig cfg

lastCheckpoint

protected org.apache.hudi.common.util.Option<String> lastCheckpoint

hoodieReadClient

protected org.apache.hudi.client.HoodieReadClient hoodieReadClient

props

protected Properties props

schema

protected String schema

configuration

protected transient org.apache.hadoop.conf.Configuration configuration

sparkContext

protected transient org.apache.spark.api.java.JavaSparkContext sparkContext

VALID_DAG_NODES_TO_ALLOW_WRITE_CLIENT_IN_DELTASTREAMER_MODE

protected static Set<String> VALID_DAG_NODES_TO_ALLOW_WRITE_CLIENT_IN_DELTASTREAMER_MODE

Constructor Detail

HoodieTestSuiteWriter

public HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc,
                             Properties props,
                             HoodieTestSuiteJob.HoodieTestSuiteConfig cfg,
                             String schema)
                      throws Exception

Throws:

Exception

Method Detail

getWriteConfig

public org.apache.hudi.config.HoodieWriteConfig getWriteConfig()

shutdownResources

public abstract void shutdownResources()

getNextBatch

public abstract org.apache.spark.rdd.RDD<org.apache.avro.generic.GenericRecord> getNextBatch()
                                                                                      throws Exception

Throws:

Exception

fetchSource

public abstract org.apache.hudi.common.util.collection.Pair<SchemaProvider,org.apache.hudi.common.util.collection.Pair<String,org.apache.spark.api.java.JavaRDD<org.apache.hudi.common.model.HoodieRecord>>> fetchSource()
                                                                                                                                                                                                                  throws Exception

Throws:

Exception

startCommit

public abstract org.apache.hudi.common.util.Option<String> startCommit()

upsert

public abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> upsert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                      throws Exception

Throws:

Exception

insert

public abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> insert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                      throws Exception

Throws:

Exception

insertOverwrite

public abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> insertOverwrite(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                               throws Exception

Throws:

Exception

insertOverwriteTable

public abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> insertOverwriteTable(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                                    throws Exception

Throws:

Exception

bulkInsert

public abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> bulkInsert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                          throws Exception

Throws:

Exception

compact

public abstract org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> compact(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                       throws Exception

Throws:

Exception

inlineClustering

public abstract void inlineClustering()
                               throws Exception

Throws:

Exception

scheduleCompaction

public abstract org.apache.hudi.common.util.Option<String> scheduleCompaction(org.apache.hudi.common.util.Option<Map<String,String>> previousCommitExtraMetadata)
                                                                       throws Exception

Throws:

Exception

commit

public abstract void commit(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records,
                            org.apache.spark.api.java.JavaRDD<DeltaWriteStats> generatedDataStats,
                            org.apache.hudi.common.util.Option<String> instantTime)

commitCompaction

public abstract void commitCompaction(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records,
                                      org.apache.spark.api.java.JavaRDD<DeltaWriteStats> generatedDataStats,
                                      org.apache.hudi.common.util.Option<String> instantTime)
                               throws Exception

Throws:

Exception

getWriteClient

public org.apache.hudi.client.SparkRDDWriteClient getWriteClient(DagNode dagNode)
                                                          throws IllegalAccessException

Throws:

IllegalAccessException

getDeltaStreamerWrapper

public HoodieDeltaStreamerWrapper getDeltaStreamerWrapper()

getCfg

public HoodieTestSuiteJob.HoodieTestSuiteConfig getCfg()

getConfiguration

public org.apache.hadoop.conf.Configuration getConfiguration()

getSparkContext

public org.apache.spark.api.java.JavaSparkContext getSparkContext()

getLastCheckpoint

public org.apache.hudi.common.util.Option<String> getLastCheckpoint()

getProps

public Properties getProps()

getSchema

public String getSchema()