org.apache.hudi.integ.testsuite

Class HoodieTestSuiteWriter

java.lang.Object

org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter

public class HoodieTestSuiteWriter
extends Object

A writer abstraction for the Hudi test suite. This class wraps different implementations of writers used to perform write operations into the target hudi dataset. Current supported writers are HoodieDeltaStreamerWrapper and HoodieWriteClient.

Field Summary

Fields

Modifier and Type	Field and Description
protected HoodieTestSuiteJob.HoodieTestSuiteConfig	cfg

Constructor Summary

Constructors

Constructor and Description
HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc, Properties props, HoodieTestSuiteJob.HoodieTestSuiteConfig cfg, String schema)
HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc, Properties props, HoodieTestSuiteJob.HoodieTestSuiteConfig cfg, String schema, boolean rollbackInflight)

Method Summary

Modifier and Type	Method and Description
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	bulkInsert(org.apache.hudi.common.util.Option<String> instantTime)
void	commit(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records, org.apache.hudi.common.util.Option<String> instantTime)
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	compact(org.apache.hudi.common.util.Option<String> instantTime)
org.apache.hudi.common.util.collection.Pair<SchemaProvider,org.apache.hudi.common.util.collection.Pair<String,org.apache.spark.api.java.JavaRDD<org.apache.hudi.common.model.HoodieRecord>>>	fetchSource()
HoodieTestSuiteJob.HoodieTestSuiteConfig	getCfg()
org.apache.hadoop.conf.Configuration	getConfiguration()
HoodieDeltaStreamerWrapper	getDeltaStreamerWrapper()
org.apache.spark.api.java.JavaSparkContext	getSparkContext()
org.apache.hudi.client.HoodieWriteClient	getWriteClient(DagNode dagNode)
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	insert(org.apache.hudi.common.util.Option<String> instantTime)
org.apache.hudi.common.util.Option<String>	scheduleCompaction(org.apache.hudi.common.util.Option<Map<String,String>> previousCommitExtraMetadata)
org.apache.hudi.common.util.Option<String>	startCommit()
org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus>	upsert(org.apache.hudi.common.util.Option<String> instantTime)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

cfg

protected HoodieTestSuiteJob.HoodieTestSuiteConfig cfg

Constructor Detail

HoodieTestSuiteWriter

public HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc,
                             Properties props,
                             HoodieTestSuiteJob.HoodieTestSuiteConfig cfg,
                             String schema)
                      throws Exception

Throws:

Exception

HoodieTestSuiteWriter

public HoodieTestSuiteWriter(org.apache.spark.api.java.JavaSparkContext jsc,
                             Properties props,
                             HoodieTestSuiteJob.HoodieTestSuiteConfig cfg,
                             String schema,
                             boolean rollbackInflight)
                      throws Exception

Throws:

Exception

Method Detail

fetchSource

public org.apache.hudi.common.util.collection.Pair<SchemaProvider,org.apache.hudi.common.util.collection.Pair<String,org.apache.spark.api.java.JavaRDD<org.apache.hudi.common.model.HoodieRecord>>> fetchSource()
                                                                                                                                                                                                         throws Exception

Throws:

Exception

startCommit

public org.apache.hudi.common.util.Option<String> startCommit()

upsert

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> upsert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                             throws Exception

Throws:

Exception

insert

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> insert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                             throws Exception

Throws:

Exception

bulkInsert

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> bulkInsert(org.apache.hudi.common.util.Option<String> instantTime)
                                                                                 throws Exception

Throws:

Exception

compact

public org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> compact(org.apache.hudi.common.util.Option<String> instantTime)
                                                                              throws Exception

Throws:

Exception

scheduleCompaction

public org.apache.hudi.common.util.Option<String> scheduleCompaction(org.apache.hudi.common.util.Option<Map<String,String>> previousCommitExtraMetadata)
                                                              throws Exception

Throws:

Exception

commit

public void commit(org.apache.spark.api.java.JavaRDD<org.apache.hudi.client.WriteStatus> records,
                   org.apache.hudi.common.util.Option<String> instantTime)

getWriteClient

public org.apache.hudi.client.HoodieWriteClient getWriteClient(DagNode dagNode)
                                                        throws IllegalAccessException

Throws:

IllegalAccessException

getDeltaStreamerWrapper

public HoodieDeltaStreamerWrapper getDeltaStreamerWrapper()

getCfg

public HoodieTestSuiteJob.HoodieTestSuiteConfig getCfg()

getConfiguration

public org.apache.hadoop.conf.Configuration getConfiguration()

getSparkContext

public org.apache.spark.api.java.JavaSparkContext getSparkContext()