Constant Field Values

Contents

• datahub.spark.*
• io.openlineage.*

datahub.spark.*

datahub.spark.conf.SparkConfigParser

Modifier and Type	Constant Field	Value
public static final String	COALESCE_KEY	"coalesce_jobs"
public static final String	CONFIG_LOG_MCPS	"log.mcps"
public static final String	DATABRICKS_CLUSTER_KEY	"databricks.cluster"
public static final String	DATAHUB_FLOW_NAME	"flow_name"
public static final String	DATASET_ENV_KEY	"metadata.dataset.env"
public static final String	DATASET_HIVE_PLATFORM_ALIAS	"metadata.dataset.hivePlatformAlias"
public static final String	DATASET_INCLUDE_SCHEMA_METADATA	"metadata.dataset.include_schema_metadata"
public static final String	DATASET_INCLUDE_SCHEMA_METADATA_DEPRECATED_ALIAS	"metadata.dataset.experimental_include_schema_metadata"
public static final String	DATASET_LOWERCASE_URNS	"metadata.dataset.lowerCaseUrns"
public static final String	DATASET_MATERIALIZE_KEY	"metadata.dataset.materialize"
public static final String	DATASET_PLATFORM_INSTANCE_KEY	"metadata.dataset.platformInstance"
public static final String	DISABLE_SSL_VERIFICATION_KEY	"rest.disable_ssl_verification"
public static final String	DISABLE_SYMLINK_RESOLUTION	"disableSymlinkResolution"
public static final String	DOMAINS_KEY	"domains"
public static final String	EMITTER_TYPE	"emitter"
public static final String	FABRIC_TYPE_KEY	"env"
public static final String	FILE_EMITTER_FILE_NAME	"file.filename"
public static final String	FILE_PARTITION_REGEXP_PATTERN	"file_partition_regexp"
public static final String	GMS_AUTH_TOKEN	"rest.token"
public static final String	GMS_URL_KEY	"rest.server"
public static final String	KAFKA_EMITTER_BOOTSTRAP	"kafka.bootstrap"
public static final String	KAFKA_EMITTER_PRODUCER_CONFIG	"kafka.producer_config"
public static final String	KAFKA_EMITTER_SCHEMA_REGISTRY_CONFIG	"kafka.schema_registry_config"
public static final String	KAFKA_EMITTER_SCHEMA_REGISTRY_URL	"kafka.schema_registry_url"
public static final String	KAFKA_MCP_TOPIC	"kafka.mcp_topic"
public static final String	LEGACY_LINEAGE_CLEANUP	"legacyLineageCleanup.enabled"
public static final String	MAX_RETRIES	"rest.max_retries"
public static final String	PARENT_JOB_KEY	"parent.datajob_urn"
public static final String	PATCH_ENABLED	"patch.enabled"
public static final String	PATH_SPEC_LIST_KEY	"path_spec_list"
public static final String	PIPELINE_KEY	"metadata.pipeline"
public static final String	PIPELINE_PLATFORM_INSTANCE_KEY	"metadata.pipeline.platformInstance"
public static final String	PLATFORM_INSTANCE_KEY	"platformInstance"
public static final String	PLATFORM_KEY	"platform"
public static final String	REMOVE_PARTITION_PATTERN	"metadata.remove_partition_pattern"
public static final String	REST_DISABLE_CHUNKED_ENCODING	"rest.disable_chunked_encoding"
public static final String	RETRY_INTERVAL_IN_SEC	"rest.retry_interval_in_sec"
public static final String	S3_EMITTER_ACCESS_KEY	"s3.access_key"
public static final String	S3_EMITTER_BUCKET	"s3.bucket"
public static final String	S3_EMITTER_ENDPOINT	"s3.endpoint"
public static final String	S3_EMITTER_FILE_NAME	"s3.filename"
public static final String	S3_EMITTER_PREFIX	"s3.prefix"
public static final String	S3_EMITTER_PROFILE	"s3.profile"
public static final String	S3_EMITTER_REGION	"s3.region"
public static final String	S3_EMITTER_SECRET_KEY	"s3.secret_key"
public static final String	SPARK_APP_NAME	"spark.app.name"
public static final String	SPARK_DATABRICKS_CLUSTER_USAGE_TAGS_CLUSTER_ALL_TAGS	"spark.databricks.clusterUsageTags.clusterAllTags"
public static final String	SPARK_MASTER	"spark.master"
public static final String	SPARK_PLATFORM_INSTANCE_KEY	"platformInstance"
public static final String	STAGE_METADATA_COALESCING	"stage_metadata_coalescing"
public static final String	STREAMING_HEARTBEAT	"streaming_heartbeat"
public static final String	STREAMING_JOB	"streaming_job"
public static final String	TAGS_KEY	"tags"

datahub.spark.converter.SparkStreamingEventToDatahub

Modifier and Type	Constant Field	Value
public static final String	DELTA_LAKE_PLATFORM	"delta-lake"
public static final String	FILE_PLATFORM	"file"
public static final String	KAFKA_PLATFORM	"kafka"

io.openlineage.*

io.openlineage.spark.agent.util.RemovePathPatternUtils

Modifier and Type	Constant Field	Value
public static final String	REMOVE_PATTERN_GROUP	"remove"
public static final String	SPARK_OPENLINEAGE_DATASET_REMOVE_PATH_PATTERN	"spark.openlineage.dataset.removePath.pattern"

io.openlineage.spark.agent.vendor.redshift.Constants

Modifier and Type	Constant Field	Value
public static final String	REDSHIFT_CLASS_NAME	"io.github.spark_redshift_community.spark.redshift.RedshiftRelation"
public static final String	REDSHIFT_PROVIDER_CLASS_NAME	"io.github.spark_redshift_community.spark.redshift.DefaultSource"

io.openlineage.spark.agent.vendor.redshift.lifecycle.RedshiftDataset

Modifier and Type	Constant Field	Value
public static final String	DEFAULT_SCHEMA	"public"
public static final String	REDSHIFT_PREFIX	"redshift://"