Package datahub.spark

Class DatahubEventEmitter

java.lang.Object

io.openlineage.spark.agent.EventEmitter

datahub.spark.DatahubEventEmitter

public class DatahubEventEmitter
extends io.openlineage.spark.agent.EventEmitter

Constructor Summary

Constructors

Constructor	Description
DatahubEventEmitter(io.openlineage.spark.api.SparkOpenLineageConfig config, String applicationJobName)

Method Summary

Modifier and Type	Method	Description
Optional<io.datahubproject.openlineage.dataset.DatahubJob>	convertOpenLineageRunEventToDatahubJob(io.openlineage.client.OpenLineage.RunEvent event)
void	emit(io.openlineage.client.OpenLineage.RunEvent event)
void	emit(org.apache.spark.sql.streaming.StreamingQueryProgress event)
void	emitCoalesced()
protected void	emitMcps(List<com.linkedin.mxe.MetadataChangeProposal> mcps)
List<com.linkedin.mxe.MetadataChangeProposal>	generateCoalescedMcps()
boolean	isStreaming()
void	setConfig(SparkLineageConf sparkConfig)
void	setStreaming(boolean enabled)

Methods inherited from class io.openlineage.spark.agent.EventEmitter

getApplicationJobName, getApplicationRunId, getClient, getCustomEnvironmentVariables, getJobNamespace, getOverriddenAppName, getParentJobName, getParentJobNamespace, getParentRunId

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

DatahubEventEmitter

public DatahubEventEmitter(io.openlineage.spark.api.SparkOpenLineageConfig config,
 String applicationJobName)
                    throws URISyntaxException

Throws:

URISyntaxException

Method Details

convertOpenLineageRunEventToDatahubJob

public Optional<io.datahubproject.openlineage.dataset.DatahubJob> convertOpenLineageRunEventToDatahubJob(io.openlineage.client.OpenLineage.RunEvent event)

emit

public void emit(io.openlineage.client.OpenLineage.RunEvent event)

Overrides:

emit in class io.openlineage.spark.agent.EventEmitter

emitCoalesced

public void emitCoalesced()

generateCoalescedMcps

public List<com.linkedin.mxe.MetadataChangeProposal> generateCoalescedMcps()

emit

public void emit(org.apache.spark.sql.streaming.StreamingQueryProgress event)
          throws URISyntaxException

Throws:

URISyntaxException

emitMcps

protected void emitMcps(List<com.linkedin.mxe.MetadataChangeProposal> mcps)

setConfig

public void setConfig(SparkLineageConf sparkConfig)

isStreaming

public boolean isStreaming()

setStreaming

public void setStreaming(boolean enabled)