public class UtilHelpers extends Object
| Constructor and Description |
|---|
UtilHelpers() |
| Modifier and Type | Method and Description |
|---|---|
static TypedProperties |
buildProperties(List<String> props) |
static org.apache.spark.api.java.JavaSparkContext |
buildSparkContext(String appName,
String defaultMaster) |
static org.apache.spark.api.java.JavaSparkContext |
buildSparkContext(String appName,
String defaultMaster,
Map<String,String> configs) |
static org.apache.spark.api.java.JavaSparkContext |
buildSparkContext(String appName,
String sparkMaster,
String sparkMemory)
Build Spark Context for ingestion/compaction
|
static HoodieWriteClient |
createHoodieClient(org.apache.spark.api.java.JavaSparkContext jsc,
String basePath,
String schemaStr,
int parallelism,
Option<String> compactionStrategyClass,
TypedProperties properties)
Build Hoodie write client
|
static SchemaProvider |
createSchemaProvider(String schemaProviderClass,
TypedProperties cfg,
org.apache.spark.api.java.JavaSparkContext jssc) |
static Source |
createSource(String sourceClass,
TypedProperties cfg,
org.apache.spark.api.java.JavaSparkContext jssc,
org.apache.spark.sql.SparkSession sparkSession,
SchemaProvider schemaProvider) |
static Transformer |
createTransformer(String transformerClass) |
static int |
handleErrors(org.apache.spark.api.java.JavaSparkContext jsc,
String instantTime,
org.apache.spark.api.java.JavaRDD<WriteStatus> writeResponse) |
static String |
parseSchema(org.apache.hadoop.fs.FileSystem fs,
String schemaFile)
Parse Schema from file
|
static DFSPropertiesConfiguration |
readConfig(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path cfgPath,
List<String> overriddenProps) |
static TypedProperties |
readConfig(InputStream in) |
public static Source createSource(String sourceClass, TypedProperties cfg, org.apache.spark.api.java.JavaSparkContext jssc, org.apache.spark.sql.SparkSession sparkSession, SchemaProvider schemaProvider) throws IOException
IOExceptionpublic static SchemaProvider createSchemaProvider(String schemaProviderClass, TypedProperties cfg, org.apache.spark.api.java.JavaSparkContext jssc) throws IOException
IOExceptionpublic static Transformer createTransformer(String transformerClass) throws IOException
IOExceptionpublic static DFSPropertiesConfiguration readConfig(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path cfgPath, List<String> overriddenProps)
public static TypedProperties buildProperties(List<String> props)
public static String parseSchema(org.apache.hadoop.fs.FileSystem fs, String schemaFile) throws Exception
fs - File SystemschemaFile - Schema FileExceptionpublic static org.apache.spark.api.java.JavaSparkContext buildSparkContext(String appName, String defaultMaster, Map<String,String> configs)
public static org.apache.spark.api.java.JavaSparkContext buildSparkContext(String appName, String defaultMaster)
public static org.apache.spark.api.java.JavaSparkContext buildSparkContext(String appName, String sparkMaster, String sparkMemory)
public static HoodieWriteClient createHoodieClient(org.apache.spark.api.java.JavaSparkContext jsc, String basePath, String schemaStr, int parallelism, Option<String> compactionStrategyClass, TypedProperties properties) throws Exception
jsc - Java Spark ContextbasePath - Base PathschemaStr - Schemaparallelism - ParallelismExceptionpublic static int handleErrors(org.apache.spark.api.java.JavaSparkContext jsc,
String instantTime,
org.apache.spark.api.java.JavaRDD<WriteStatus> writeResponse)
public static TypedProperties readConfig(InputStream in) throws IOException
IOExceptionCopyright © 2019 The Apache Software Foundation. All rights reserved.