Package datahub.spark.conf

Class SparkConfigParser

java.lang.Object

datahub.spark.conf.SparkConfigParser

public class SparkConfigParser
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
static final String	COALESCE_KEY
static final String	CONFIG_LOG_MCPS
static final String	DATABRICKS_CLUSTER_KEY
static final String	DATAHUB_FLOW_NAME
static final String	DATASET_ENV_KEY
static final String	DATASET_HIVE_PLATFORM_ALIAS
static final String	DATASET_INCLUDE_SCHEMA_METADATA
static final String	DATASET_INCLUDE_SCHEMA_METADATA_DEPRECATED_ALIAS
static final String	DATASET_LOWERCASE_URNS
static final String	DATASET_MATERIALIZE_KEY
static final String	DATASET_PLATFORM_INSTANCE_KEY
static final String	DISABLE_SSL_VERIFICATION_KEY
static final String	DISABLE_SYMLINK_RESOLUTION
static final String	DOMAINS_KEY
static final String	EMITTER_TYPE
static final String	FABRIC_TYPE_KEY
static final String	FILE_EMITTER_FILE_NAME
static final String	FILE_PARTITION_REGEXP_PATTERN
static final String	GMS_AUTH_TOKEN
static final String	GMS_URL_KEY
static final String	KAFKA_EMITTER_BOOTSTRAP
static final String	KAFKA_EMITTER_PRODUCER_CONFIG
static final String	KAFKA_EMITTER_SCHEMA_REGISTRY_CONFIG
static final String	KAFKA_EMITTER_SCHEMA_REGISTRY_URL
static final String	KAFKA_MCP_TOPIC
static final String	LEGACY_LINEAGE_CLEANUP
static final String	MAX_RETRIES
static final String	PARENT_JOB_KEY
static final String	PATCH_ENABLED
static final String	PATH_SPEC_LIST_KEY
static final String	PIPELINE_KEY
static final String	PIPELINE_PLATFORM_INSTANCE_KEY
static final String	PLATFORM_INSTANCE_KEY
static final String	PLATFORM_KEY
static final String	REMOVE_PARTITION_PATTERN
static final String	REST_DISABLE_CHUNKED_ENCODING
static final String	RETRY_INTERVAL_IN_SEC
static final String	S3_EMITTER_ACCESS_KEY
static final String	S3_EMITTER_BUCKET
static final String	S3_EMITTER_ENDPOINT
static final String	S3_EMITTER_FILE_NAME
static final String	S3_EMITTER_PREFIX
static final String	S3_EMITTER_PROFILE
static final String	S3_EMITTER_REGION
static final String	S3_EMITTER_SECRET_KEY
static final String	SPARK_APP_NAME
static final String	SPARK_DATABRICKS_CLUSTER_USAGE_TAGS_CLUSTER_ALL_TAGS
static final String	SPARK_MASTER
static final String	SPARK_PLATFORM_INSTANCE_KEY
static final String	STAGE_METADATA_COALESCING
static final String	STREAMING_HEARTBEAT
static final String	STREAMING_JOB
static final String	TAGS_KEY

Method Summary

Modifier and Type	Method	Description
static com.linkedin.common.FabricType	getCommonFabricType(com.typesafe.config.Config datahubConfig)
static String	getCommonPlatformInstance(com.typesafe.config.Config datahubConfig)
static Optional<Map<String,String>>	getDatabricksClusterTags(String databricksClusterTags)
static Optional<Map<String,String>>	getDatabricksTags(com.typesafe.config.Config datahubConfig)
static String[]	getDomains(com.typesafe.config.Config datahubConfig)
static String	getFilePartitionRegexpPattern(com.typesafe.config.Config config)
static String	getHivePlatformAlias(com.typesafe.config.Config datahubConfig)
static String	getParentJobKey(com.typesafe.config.Config datahubConfig)
static Map<String,List<io.datahubproject.openlineage.dataset.PathSpec>>	getPathSpecListMap(com.typesafe.config.Config datahubConfig)
static String	getPipelineName(com.typesafe.config.Config datahubConfig, SparkAppContext appContext)
static String	getPlatformInstance(com.typesafe.config.Config pathSpecConfig)
static String	getRemovePartitionPattern(com.typesafe.config.Config datahubConfig)
static String	getSparkAppName(com.typesafe.config.Config datahubConfig)
static String	getSparkMaster(com.typesafe.config.Config datahubConfig)
static int	getStreamingHeartbeatSec(com.typesafe.config.Config datahubConfig)
static String[]	getTags(com.typesafe.config.Config datahubConfig)
static boolean	isCoalesceEnabled(com.typesafe.config.Config datahubConfig)
static boolean	isDatasetMaterialize(com.typesafe.config.Config datahubConfig)
static boolean	isDisableSymlinkResolution(com.typesafe.config.Config datahubConfig)
static boolean	isEmitCoalescePeriodically(com.typesafe.config.Config datahubConfig)
static boolean	isIncludeSchemaMetadata(com.typesafe.config.Config datahubConfig)
static boolean	isLegacyLineageCleanupEnabled(com.typesafe.config.Config datahubConfig)
static boolean	isLogMcps(com.typesafe.config.Config datahubConfig)
static boolean	isLowerCaseDatasetUrns(com.typesafe.config.Config datahubConfig)
static boolean	isPatchEnabled(com.typesafe.config.Config datahubConfig)
static Properties	moveKeysToRoot(Properties properties, String prefix)
static com.typesafe.config.Config	parsePropertiesToConfig(Properties properties)
static com.typesafe.config.Config	parseSparkConfig()
static io.datahubproject.openlineage.config.DatahubOpenlineageConfig	sparkConfigToDatahubOpenlineageConf(com.typesafe.config.Config sparkConfig, SparkAppContext sparkAppContext)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

PARENT_JOB_KEY

public static final String PARENT_JOB_KEY

See Also:

• Constant Field Values

EMITTER_TYPE

public static final String EMITTER_TYPE

See Also:

• Constant Field Values

GMS_URL_KEY

public static final String GMS_URL_KEY

See Also:

• Constant Field Values

GMS_AUTH_TOKEN

public static final String GMS_AUTH_TOKEN

See Also:

• Constant Field Values

FILE_EMITTER_FILE_NAME

public static final String FILE_EMITTER_FILE_NAME

See Also:

• Constant Field Values

DISABLE_SSL_VERIFICATION_KEY

public static final String DISABLE_SSL_VERIFICATION_KEY

See Also:

• Constant Field Values

REST_DISABLE_CHUNKED_ENCODING

public static final String REST_DISABLE_CHUNKED_ENCODING

See Also:

• Constant Field Values

CONFIG_LOG_MCPS

public static final String CONFIG_LOG_MCPS

See Also:

• Constant Field Values

MAX_RETRIES

public static final String MAX_RETRIES

See Also:

• Constant Field Values

RETRY_INTERVAL_IN_SEC

public static final String RETRY_INTERVAL_IN_SEC

See Also:

• Constant Field Values

KAFKA_MCP_TOPIC

public static final String KAFKA_MCP_TOPIC

See Also:

• Constant Field Values

KAFKA_EMITTER_BOOTSTRAP

public static final String KAFKA_EMITTER_BOOTSTRAP

See Also:

• Constant Field Values

KAFKA_EMITTER_SCHEMA_REGISTRY_URL

public static final String KAFKA_EMITTER_SCHEMA_REGISTRY_URL

See Also:

• Constant Field Values

KAFKA_EMITTER_SCHEMA_REGISTRY_CONFIG

public static final String KAFKA_EMITTER_SCHEMA_REGISTRY_CONFIG

See Also:

• Constant Field Values

KAFKA_EMITTER_PRODUCER_CONFIG

public static final String KAFKA_EMITTER_PRODUCER_CONFIG

See Also:

• Constant Field Values

S3_EMITTER_BUCKET

public static final String S3_EMITTER_BUCKET

See Also:

• Constant Field Values

S3_EMITTER_REGION

public static final String S3_EMITTER_REGION

See Also:

• Constant Field Values

S3_EMITTER_ENDPOINT

public static final String S3_EMITTER_ENDPOINT

See Also:

• Constant Field Values

S3_EMITTER_PREFIX

public static final String S3_EMITTER_PREFIX

See Also:

• Constant Field Values

S3_EMITTER_ACCESS_KEY

public static final String S3_EMITTER_ACCESS_KEY

See Also:

• Constant Field Values

S3_EMITTER_SECRET_KEY

public static final String S3_EMITTER_SECRET_KEY

See Also:

• Constant Field Values

S3_EMITTER_PROFILE

public static final String S3_EMITTER_PROFILE

See Also:

• Constant Field Values

S3_EMITTER_FILE_NAME

public static final String S3_EMITTER_FILE_NAME

See Also:

• Constant Field Values

COALESCE_KEY

public static final String COALESCE_KEY

See Also:

• Constant Field Values

PATCH_ENABLED

public static final String PATCH_ENABLED

See Also:

• Constant Field Values

LEGACY_LINEAGE_CLEANUP

public static final String LEGACY_LINEAGE_CLEANUP

See Also:

• Constant Field Values

DISABLE_SYMLINK_RESOLUTION

public static final String DISABLE_SYMLINK_RESOLUTION

See Also:

• Constant Field Values

STAGE_METADATA_COALESCING

public static final String STAGE_METADATA_COALESCING

See Also:

• Constant Field Values

STREAMING_JOB

public static final String STREAMING_JOB

See Also:

• Constant Field Values

STREAMING_HEARTBEAT

public static final String STREAMING_HEARTBEAT

See Also:

• Constant Field Values

DATAHUB_FLOW_NAME

public static final String DATAHUB_FLOW_NAME

See Also:

• Constant Field Values

DATASET_ENV_KEY

public static final String DATASET_ENV_KEY

See Also:

• Constant Field Values

DATASET_HIVE_PLATFORM_ALIAS

public static final String DATASET_HIVE_PLATFORM_ALIAS

See Also:

• Constant Field Values

DATASET_LOWERCASE_URNS

public static final String DATASET_LOWERCASE_URNS

See Also:

• Constant Field Values

DATASET_MATERIALIZE_KEY

public static final String DATASET_MATERIALIZE_KEY

See Also:

• Constant Field Values

DATASET_PLATFORM_INSTANCE_KEY

public static final String DATASET_PLATFORM_INSTANCE_KEY

See Also:

• Constant Field Values

DATASET_INCLUDE_SCHEMA_METADATA_DEPRECATED_ALIAS

public static final String DATASET_INCLUDE_SCHEMA_METADATA_DEPRECATED_ALIAS

See Also:

• Constant Field Values

DATASET_INCLUDE_SCHEMA_METADATA

public static final String DATASET_INCLUDE_SCHEMA_METADATA

See Also:

• Constant Field Values

SPARK_PLATFORM_INSTANCE_KEY

public static final String SPARK_PLATFORM_INSTANCE_KEY

See Also:

• Constant Field Values

REMOVE_PARTITION_PATTERN

public static final String REMOVE_PARTITION_PATTERN

See Also:

• Constant Field Values

SPARK_APP_NAME

public static final String SPARK_APP_NAME

See Also:

• Constant Field Values

SPARK_MASTER

public static final String SPARK_MASTER

See Also:

• Constant Field Values

PLATFORM_KEY

public static final String PLATFORM_KEY

See Also:

• Constant Field Values

PATH_SPEC_LIST_KEY

public static final String PATH_SPEC_LIST_KEY

See Also:

• Constant Field Values

FILE_PARTITION_REGEXP_PATTERN

public static final String FILE_PARTITION_REGEXP_PATTERN

See Also:

• Constant Field Values

FABRIC_TYPE_KEY

public static final String FABRIC_TYPE_KEY

See Also:

• Constant Field Values

PLATFORM_INSTANCE_KEY

public static final String PLATFORM_INSTANCE_KEY

See Also:

• Constant Field Values

DATABRICKS_CLUSTER_KEY

public static final String DATABRICKS_CLUSTER_KEY

See Also:

• Constant Field Values

PIPELINE_KEY

public static final String PIPELINE_KEY

See Also:

• Constant Field Values

PIPELINE_PLATFORM_INSTANCE_KEY

public static final String PIPELINE_PLATFORM_INSTANCE_KEY

See Also:

• Constant Field Values

TAGS_KEY

public static final String TAGS_KEY

See Also:

• Constant Field Values

DOMAINS_KEY

public static final String DOMAINS_KEY

See Also:

• Constant Field Values

SPARK_DATABRICKS_CLUSTER_USAGE_TAGS_CLUSTER_ALL_TAGS

public static final String SPARK_DATABRICKS_CLUSTER_USAGE_TAGS_CLUSTER_ALL_TAGS

See Also:

• Constant Field Values

Method Details

moveKeysToRoot

public static Properties moveKeysToRoot(Properties properties,
 String prefix)

parsePropertiesToConfig

public static com.typesafe.config.Config parsePropertiesToConfig(Properties properties)

parseSparkConfig

public static com.typesafe.config.Config parseSparkConfig()

getDatabricksClusterTags

public static Optional<Map<String,String>> getDatabricksClusterTags(String databricksClusterTags)

sparkConfigToDatahubOpenlineageConf

public static io.datahubproject.openlineage.config.DatahubOpenlineageConfig sparkConfigToDatahubOpenlineageConf(com.typesafe.config.Config sparkConfig,
 SparkAppContext sparkAppContext)

getCommonFabricType

public static com.linkedin.common.FabricType getCommonFabricType(com.typesafe.config.Config datahubConfig)

getHivePlatformAlias

public static String getHivePlatformAlias(com.typesafe.config.Config datahubConfig)

getCommonPlatformInstance

public static String getCommonPlatformInstance(com.typesafe.config.Config datahubConfig)

getDatabricksTags

public static Optional<Map<String,String>> getDatabricksTags(com.typesafe.config.Config datahubConfig)

getParentJobKey

public static String getParentJobKey(com.typesafe.config.Config datahubConfig)

getTags

public static String[] getTags(com.typesafe.config.Config datahubConfig)

getDomains

public static String[] getDomains(com.typesafe.config.Config datahubConfig)

getSparkMaster

public static String getSparkMaster(com.typesafe.config.Config datahubConfig)

getRemovePartitionPattern

public static String getRemovePartitionPattern(com.typesafe.config.Config datahubConfig)

getSparkAppName

public static String getSparkAppName(com.typesafe.config.Config datahubConfig)

getPathSpecListMap

public static Map<String,List<io.datahubproject.openlineage.dataset.PathSpec>> getPathSpecListMap(com.typesafe.config.Config datahubConfig)

getPlatformInstance

public static String getPlatformInstance(com.typesafe.config.Config pathSpecConfig)

getFilePartitionRegexpPattern

public static String getFilePartitionRegexpPattern(com.typesafe.config.Config config)

getStreamingHeartbeatSec

public static int getStreamingHeartbeatSec(com.typesafe.config.Config datahubConfig)

isDatasetMaterialize

public static boolean isDatasetMaterialize(com.typesafe.config.Config datahubConfig)

isLogMcps

public static boolean isLogMcps(com.typesafe.config.Config datahubConfig)

isIncludeSchemaMetadata

public static boolean isIncludeSchemaMetadata(com.typesafe.config.Config datahubConfig)

getPipelineName

public static String getPipelineName(com.typesafe.config.Config datahubConfig,
 SparkAppContext appContext)

isCoalesceEnabled

public static boolean isCoalesceEnabled(com.typesafe.config.Config datahubConfig)

isPatchEnabled

public static boolean isPatchEnabled(com.typesafe.config.Config datahubConfig)

isLegacyLineageCleanupEnabled

public static boolean isLegacyLineageCleanupEnabled(com.typesafe.config.Config datahubConfig)

isDisableSymlinkResolution

public static boolean isDisableSymlinkResolution(com.typesafe.config.Config datahubConfig)

isEmitCoalescePeriodically

public static boolean isEmitCoalescePeriodically(com.typesafe.config.Config datahubConfig)

isLowerCaseDatasetUrns

public static boolean isLowerCaseDatasetUrns(com.typesafe.config.Config datahubConfig)