Package org.apache.beam.runners.spark.structuredstreaming.translation

Class TransformTranslator.Context

java.lang.Object

org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator.Context

All Implemented Interfaces:

EncoderProvider, PipelineTranslator.TranslationState

Enclosing class:

TransformTranslator<InT extends org.apache.beam.sdk.values.PInput,OutT extends org.apache.beam.sdk.values.POutput,TransformT extends org.apache.beam.sdk.transforms.PTransform<InT,OutT>>

protected class TransformTranslator.Context
extends java.lang.Object
implements PipelineTranslator.TranslationState

Available mutable context to translate a PTransform. The context is backed by the shared PipelineTranslator.TranslationState of the PipelineTranslator.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

EncoderProvider.Factory<T>

Method Summary

Modifier and Type	Method	Description
<T> org.apache.spark.broadcast.Broadcast<T>	broadcast(T value)
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data, org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)
<T> org.apache.spark.sql.Encoder<T>	encoderOf(org.apache.beam.sdk.coders.Coder<T> coder, EncoderProvider.Factory<T> factory)
org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT>	getCurrentTransform()
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	getDataset(org.apache.beam.sdk.values.PCollection<T> pCollection)
InT	getInput()
java.util.Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>>	getInputs()
org.apache.beam.sdk.options.PipelineOptions	getOptions()
java.util.function.Supplier<org.apache.beam.sdk.options.PipelineOptions>	getOptionsSupplier()
OutT	getOutput()
<T> org.apache.beam.sdk.values.PCollection<T>	getOutput(org.apache.beam.sdk.values.TupleTag<T> tag)
java.util.Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>>	getOutputs()
<T> org.apache.spark.broadcast.Broadcast<SideInputValues<T>>	getSideInputBroadcast(org.apache.beam.sdk.values.PCollection<T> pCollection, SideInputValues.Loader<T> loader)
org.apache.spark.sql.SparkSession	getSparkSession()
boolean	isLeaf(org.apache.beam.sdk.values.PCollection<?> pCollection)
<T> void	putDataset(org.apache.beam.sdk.values.PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset, boolean cache)
<InputT,T> void	putUnresolved(org.apache.beam.sdk.values.PCollection<T> out, PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)
<T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>>	tupleEncoder(org.apache.spark.sql.Encoder<T1> e1, org.apache.spark.sql.Encoder<T2> e2)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder)
<T,W extends org.apache.beam.sdk.transforms.windowing.BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder, org.apache.beam.sdk.coders.Coder<W> windowCoder)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(org.apache.spark.sql.Encoder<T> enc)
org.apache.spark.sql.Encoder<org.apache.beam.sdk.transforms.windowing.BoundedWindow>	windowEncoder()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

encoderFactory, encoderOf, keyEncoderOf, kvEncoderOf, valueEncoderOf

Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator.TranslationState

putDataset

Method Detail

getInput

public InT getInput()

getInputs

public java.util.Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>> getInputs()

getOutputs

public java.util.Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>> getOutputs()

getOutput

public OutT getOutput()

getOutput

public <T> org.apache.beam.sdk.values.PCollection<T> getOutput(org.apache.beam.sdk.values.TupleTag<T> tag)

getCurrentTransform

public org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT> getCurrentTransform()

getDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> getDataset(org.apache.beam.sdk.values.PCollection<T> pCollection)

Specified by:

getDataset in interface PipelineTranslator.TranslationState

getSideInputBroadcast

public <T> org.apache.spark.broadcast.Broadcast<SideInputValues<T>> getSideInputBroadcast(org.apache.beam.sdk.values.PCollection<T> pCollection,
                                                                                          SideInputValues.Loader<T> loader)

Specified by:

getSideInputBroadcast in interface PipelineTranslator.TranslationState

putDataset

public <T> void putDataset(org.apache.beam.sdk.values.PCollection<T> pCollection,
                           org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset,
                           boolean cache)

Specified by:

putDataset in interface PipelineTranslator.TranslationState

putUnresolved

public <InputT,T> void putUnresolved(org.apache.beam.sdk.values.PCollection<T> out,
                                           PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)

Specified by:

putUnresolved in interface PipelineTranslator.TranslationState

isLeaf

public boolean isLeaf(org.apache.beam.sdk.values.PCollection<?> pCollection)

Specified by:

isLeaf in interface PipelineTranslator.TranslationState

getOptionsSupplier

public java.util.function.Supplier<org.apache.beam.sdk.options.PipelineOptions> getOptionsSupplier()

Specified by:

getOptionsSupplier in interface PipelineTranslator.TranslationState

getOptions

public org.apache.beam.sdk.options.PipelineOptions getOptions()

Specified by:

getOptions in interface PipelineTranslator.TranslationState

createDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data,
                                                                                                 org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)

broadcast

public <T> org.apache.spark.broadcast.Broadcast<T> broadcast(T value)

getSparkSession

public org.apache.spark.sql.SparkSession getSparkSession()

Specified by:

getSparkSession in interface PipelineTranslator.TranslationState

encoderOf

public <T> org.apache.spark.sql.Encoder<T> encoderOf(org.apache.beam.sdk.coders.Coder<T> coder,
                                                     EncoderProvider.Factory<T> factory)

Specified by:

encoderOf in interface EncoderProvider

tupleEncoder

public <T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>> tupleEncoder(org.apache.spark.sql.Encoder<T1> e1,
                                                                                          org.apache.spark.sql.Encoder<T2> e2)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(org.apache.spark.sql.Encoder<T> enc)

windowedEncoder

public <T,W extends org.apache.beam.sdk.transforms.windowing.BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder,
                                                                                                                                                                          org.apache.beam.sdk.coders.Coder<W> windowCoder)

windowEncoder

public org.apache.spark.sql.Encoder<org.apache.beam.sdk.transforms.windowing.BoundedWindow> windowEncoder()