org.apache.hudi.integ.testsuite.generator

Class DeltaGenerator

java.lang.Object

org.apache.hudi.integ.testsuite.generator.DeltaGenerator

All Implemented Interfaces:

Serializable

public class DeltaGenerator
extends Object
implements Serializable

The delta generator generates all types of workloads (insert, update) for the given configs.

See Also:

Serialized Form

Constructor Summary

Constructors

Constructor and Description
DeltaGenerator(DFSDeltaConfig deltaOutputConfig, org.apache.spark.api.java.JavaSparkContext jsc, org.apache.spark.sql.SparkSession sparkSession, String schemaStr, org.apache.hudi.keygen.BuiltinKeyGenerator keyGenerator)

Method Summary

Modifier and Type	Method and Description
org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord>	adjustRDDToGenerateExactNumUpdates(org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> updates, org.apache.spark.api.java.JavaSparkContext jsc, long totalRecordsRequired)
org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord>	generateDeletes(DeltaConfig.Config config)
org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord>	generateInserts(DeltaConfig.Config operation)
org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord>	generateUpdates(DeltaConfig.Config config)
Map<Integer,Long>	getAdjustedPartitionsCount(Map<Integer,Long> partitionCountMap, long recordsToRemove)
int	getBatchId()
Map<Integer,Long>	getPartitionToCountMap(org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> records)
org.apache.hudi.common.util.collection.Pair<Integer,org.apache.spark.api.java.JavaRDD<DeltaWriteStats>>	writeRecords(org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> records)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

DeltaGenerator

public DeltaGenerator(DFSDeltaConfig deltaOutputConfig,
                      org.apache.spark.api.java.JavaSparkContext jsc,
                      org.apache.spark.sql.SparkSession sparkSession,
                      String schemaStr,
                      org.apache.hudi.keygen.BuiltinKeyGenerator keyGenerator)

Method Detail

writeRecords

public org.apache.hudi.common.util.collection.Pair<Integer,org.apache.spark.api.java.JavaRDD<DeltaWriteStats>> writeRecords(org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> records)

getBatchId

public int getBatchId()

generateInserts

public org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> generateInserts(DeltaConfig.Config operation)

generateUpdates

public org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> generateUpdates(DeltaConfig.Config config)
                                                                                         throws IOException

Throws:

IOException

generateDeletes

public org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> generateDeletes(DeltaConfig.Config config)
                                                                                         throws IOException

Throws:

IOException

getPartitionToCountMap

public Map<Integer,Long> getPartitionToCountMap(org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> records)

getAdjustedPartitionsCount

public Map<Integer,Long> getAdjustedPartitionsCount(Map<Integer,Long> partitionCountMap,
                                                    long recordsToRemove)

adjustRDDToGenerateExactNumUpdates

public org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> adjustRDDToGenerateExactNumUpdates(org.apache.spark.api.java.JavaRDD<org.apache.avro.generic.GenericRecord> updates,
                                                                                                                   org.apache.spark.api.java.JavaSparkContext jsc,
                                                                                                                   long totalRecordsRequired)