org.apache.hudi.utilities

Class UtilHelpers

java.lang.Object

org.apache.hudi.utilities.UtilHelpers

public class UtilHelpers
extends Object

Bunch of helper methods

Constructor Summary

Constructors

Constructor and Description
UtilHelpers()

Method Summary

Modifier and Type	Method and Description
static TypedProperties	buildProperties(List<String> props)
static org.apache.spark.api.java.JavaSparkContext	buildSparkContext(String appName, String defaultMaster)
static org.apache.spark.api.java.JavaSparkContext	buildSparkContext(String appName, String defaultMaster, Map<String,String> configs)
static org.apache.spark.api.java.JavaSparkContext	buildSparkContext(String appName, String sparkMaster, String sparkMemory) Build Spark Context for ingestion/compaction
static HoodieWriteClient	createHoodieClient(org.apache.spark.api.java.JavaSparkContext jsc, String basePath, String schemaStr, int parallelism, Option<String> compactionStrategyClass, TypedProperties properties) Build Hoodie write client
static SchemaProvider	createSchemaProvider(String schemaProviderClass, TypedProperties cfg, org.apache.spark.api.java.JavaSparkContext jssc)
static Source	createSource(String sourceClass, TypedProperties cfg, org.apache.spark.api.java.JavaSparkContext jssc, org.apache.spark.sql.SparkSession sparkSession, SchemaProvider schemaProvider)
static Transformer	createTransformer(String transformerClass)
static int	handleErrors(org.apache.spark.api.java.JavaSparkContext jsc, String instantTime, org.apache.spark.api.java.JavaRDD<WriteStatus> writeResponse)
static String	parseSchema(org.apache.hadoop.fs.FileSystem fs, String schemaFile) Parse Schema from file
static DFSPropertiesConfiguration	readConfig(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path cfgPath, List<String> overriddenProps)
static TypedProperties	readConfig(InputStream in)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

UtilHelpers

public UtilHelpers()

Method Detail

createSource

public static Source createSource(String sourceClass,
                                  TypedProperties cfg,
                                  org.apache.spark.api.java.JavaSparkContext jssc,
                                  org.apache.spark.sql.SparkSession sparkSession,
                                  SchemaProvider schemaProvider)
                           throws IOException

Throws:

IOException

createSchemaProvider

public static SchemaProvider createSchemaProvider(String schemaProviderClass,
                                                  TypedProperties cfg,
                                                  org.apache.spark.api.java.JavaSparkContext jssc)
                                           throws IOException

Throws:

IOException

createTransformer

public static Transformer createTransformer(String transformerClass)
                                     throws IOException

Throws:

IOException

readConfig

public static DFSPropertiesConfiguration readConfig(org.apache.hadoop.fs.FileSystem fs,
                                                    org.apache.hadoop.fs.Path cfgPath,
                                                    List<String> overriddenProps)

buildProperties

public static TypedProperties buildProperties(List<String> props)

parseSchema

public static String parseSchema(org.apache.hadoop.fs.FileSystem fs,
                                 String schemaFile)
                          throws Exception

Parse Schema from file

Parameters:

fs - File System

schemaFile - Schema File

Throws:

Exception

buildSparkContext

public static org.apache.spark.api.java.JavaSparkContext buildSparkContext(String appName,
                                                                           String defaultMaster,
                                                                           Map<String,String> configs)

buildSparkContext

public static org.apache.spark.api.java.JavaSparkContext buildSparkContext(String appName,
                                                                           String defaultMaster)

buildSparkContext

public static org.apache.spark.api.java.JavaSparkContext buildSparkContext(String appName,
                                                                           String sparkMaster,
                                                                           String sparkMemory)

Build Spark Context for ingestion/compaction

Returns:

createHoodieClient

public static HoodieWriteClient createHoodieClient(org.apache.spark.api.java.JavaSparkContext jsc,
                                                   String basePath,
                                                   String schemaStr,
                                                   int parallelism,
                                                   Option<String> compactionStrategyClass,
                                                   TypedProperties properties)
                                            throws Exception

Build Hoodie write client

Parameters:

jsc - Java Spark Context

basePath - Base Path

schemaStr - Schema

parallelism - Parallelism

Throws:

Exception

handleErrors

public static int handleErrors(org.apache.spark.api.java.JavaSparkContext jsc,
                               String instantTime,
                               org.apache.spark.api.java.JavaRDD<WriteStatus> writeResponse)

readConfig

public static TypedProperties readConfig(InputStream in)
                                  throws IOException

Throws:

IOException