org.apache.beam.runners.samza.translation

Class TranslationContext

java.lang.Object

org.apache.beam.runners.samza.translation.TranslationContext

Direct Known Subclasses:

PortableTranslationContext

public class TranslationContext
extends java.lang.Object

Helper that keeps the mapping from BEAM PValue/PCollectionView to Samza MessageStream. It also provides other context data such as input and output of a PTransform.

Constructor Summary

Constructors

Constructor and Description
TranslationContext(org.apache.samza.application.descriptors.StreamApplicationDescriptor appDescriptor, java.util.Map<org.apache.beam.sdk.values.PValue,java.lang.String> idMap, java.util.Set<java.lang.String> nonUniqueStateIds, SamzaPipelineOptions options)

Method Summary

Modifier and Type	Method and Description
<InT extends org.apache.beam.sdk.values.PValue,OutT extends org.apache.beam.sdk.values.PValue> void	attachTransformMetricOp(org.apache.beam.sdk.transforms.PTransform<InT,OutT> transform, org.apache.beam.sdk.runners.TransformHierarchy.Node node, SamzaMetricOpFactory.OpType opType)
void	clearCurrentTransform()
org.apache.beam.sdk.runners.AppliedPTransform<?,?,?>	getCurrentTransform()
org.apache.samza.operators.MessageStream<OpMessage<java.lang.String>>	getDummyStream()
java.lang.String	getIdForPValue(org.apache.beam.sdk.values.PValue pvalue)
<InT extends org.apache.beam.sdk.values.PValue> InT	getInput(org.apache.beam.sdk.transforms.PTransform<InT,?> transform)
<OutT> org.apache.samza.operators.MessageStream<OpMessage<OutT>>	getMessageStream(org.apache.beam.sdk.values.PValue pvalue)
<OutT extends org.apache.beam.sdk.values.PValue> OutT	getOutput(org.apache.beam.sdk.transforms.PTransform<?,OutT> transform)
<OutT> org.apache.samza.operators.OutputStream<OutT>	getOutputStream(org.apache.samza.system.descriptors.OutputDescriptor<OutT,?> outputDescriptor)
<OutT> org.apache.beam.sdk.values.TupleTag<OutT>	getOutputTag(org.apache.beam.sdk.transforms.PTransform<?,? extends org.apache.beam.sdk.values.PCollection<OutT>> transform)
SamzaPipelineOptions	getPipelineOptions()
StoreIdGenerator	getStoreIdGenerator()
<K,V> org.apache.samza.table.Table<org.apache.samza.operators.KV<K,V>>	getTable(org.apache.samza.table.descriptors.TableDescriptor<K,V,?> tableDesc)
java.lang.String	getTransformFullName()
java.lang.String	getTransformId()
<ViewT> java.lang.String	getViewId(org.apache.beam.sdk.values.PCollectionView<ViewT> view)
<InT> org.apache.samza.operators.MessageStream<OpMessage<InT>>	getViewStream(org.apache.beam.sdk.values.PCollectionView<?> view)
<OutT> void	registerInputMessageStream(org.apache.beam.sdk.values.PValue pvalue, org.apache.samza.system.descriptors.InputDescriptor<org.apache.samza.operators.KV<?,OpMessage<OutT>>,?> inputDescriptor)
protected <KeyT,OutT> void	registerInputMessageStreams(KeyT key, java.util.List<? extends org.apache.samza.system.descriptors.InputDescriptor<org.apache.samza.operators.KV<?,OpMessage<OutT>>,?>> inputDescriptors, java.util.function.BiConsumer<KeyT,org.apache.samza.operators.MessageStream<OpMessage<OutT>>> registerFunction)
<OutT> void	registerInputMessageStreams(org.apache.beam.sdk.values.PValue pvalue, java.util.List<? extends org.apache.samza.system.descriptors.InputDescriptor<org.apache.samza.operators.KV<?,OpMessage<OutT>>,?>> inputDescriptors) Function to register a merged messageStream of all input messageStreams to a PCollection.
<OutT> void	registerMessageStream(org.apache.beam.sdk.values.PValue pvalue, org.apache.samza.operators.MessageStream<OpMessage<OutT>> stream)
<ElemT,ViewT> void	registerViewStream(org.apache.beam.sdk.values.PCollectionView<ViewT> view, org.apache.samza.operators.MessageStream<OpMessage<java.lang.Iterable<ElemT>>> stream)
void	setCurrentTransform(org.apache.beam.sdk.runners.AppliedPTransform<?,?,?> currentTransform)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TranslationContext

public TranslationContext(org.apache.samza.application.descriptors.StreamApplicationDescriptor appDescriptor,
                          java.util.Map<org.apache.beam.sdk.values.PValue,java.lang.String> idMap,
                          java.util.Set<java.lang.String> nonUniqueStateIds,
                          SamzaPipelineOptions options)

Method Detail

registerInputMessageStream

public <OutT> void registerInputMessageStream(org.apache.beam.sdk.values.PValue pvalue,
                                              org.apache.samza.system.descriptors.InputDescriptor<org.apache.samza.operators.KV<?,OpMessage<OutT>>,?> inputDescriptor)

registerInputMessageStreams

public <OutT> void registerInputMessageStreams(org.apache.beam.sdk.values.PValue pvalue,
                                               java.util.List<? extends org.apache.samza.system.descriptors.InputDescriptor<org.apache.samza.operators.KV<?,OpMessage<OutT>>,?>> inputDescriptors)

Function to register a merged messageStream of all input messageStreams to a PCollection.

Parameters:

pvalue - output of a transform

inputDescriptors - a list of Samza InputDescriptors

registerInputMessageStreams

protected <KeyT,OutT> void registerInputMessageStreams(KeyT key,
                                                       java.util.List<? extends org.apache.samza.system.descriptors.InputDescriptor<org.apache.samza.operators.KV<?,OpMessage<OutT>>,?>> inputDescriptors,
                                                       java.util.function.BiConsumer<KeyT,org.apache.samza.operators.MessageStream<OpMessage<OutT>>> registerFunction)

registerMessageStream

public <OutT> void registerMessageStream(org.apache.beam.sdk.values.PValue pvalue,
                                         org.apache.samza.operators.MessageStream<OpMessage<OutT>> stream)

getDummyStream

public org.apache.samza.operators.MessageStream<OpMessage<java.lang.String>> getDummyStream()

getMessageStream

public <OutT> org.apache.samza.operators.MessageStream<OpMessage<OutT>> getMessageStream(org.apache.beam.sdk.values.PValue pvalue)

attachTransformMetricOp

public <InT extends org.apache.beam.sdk.values.PValue,OutT extends org.apache.beam.sdk.values.PValue> void attachTransformMetricOp(org.apache.beam.sdk.transforms.PTransform<InT,OutT> transform,
                                                                                                                                   org.apache.beam.sdk.runners.TransformHierarchy.Node node,
                                                                                                                                   SamzaMetricOpFactory.OpType opType)

registerViewStream

public <ElemT,ViewT> void registerViewStream(org.apache.beam.sdk.values.PCollectionView<ViewT> view,
                                             org.apache.samza.operators.MessageStream<OpMessage<java.lang.Iterable<ElemT>>> stream)

getViewStream

public <InT> org.apache.samza.operators.MessageStream<OpMessage<InT>> getViewStream(org.apache.beam.sdk.values.PCollectionView<?> view)

getViewId

public <ViewT> java.lang.String getViewId(org.apache.beam.sdk.values.PCollectionView<ViewT> view)

setCurrentTransform

public void setCurrentTransform(org.apache.beam.sdk.runners.AppliedPTransform<?,?,?> currentTransform)

clearCurrentTransform

public void clearCurrentTransform()

getCurrentTransform

public org.apache.beam.sdk.runners.AppliedPTransform<?,?,?> getCurrentTransform()

getInput

public <InT extends org.apache.beam.sdk.values.PValue> InT getInput(org.apache.beam.sdk.transforms.PTransform<InT,?> transform)

getOutput

public <OutT extends org.apache.beam.sdk.values.PValue> OutT getOutput(org.apache.beam.sdk.transforms.PTransform<?,OutT> transform)

getOutputTag

public <OutT> org.apache.beam.sdk.values.TupleTag<OutT> getOutputTag(org.apache.beam.sdk.transforms.PTransform<?,? extends org.apache.beam.sdk.values.PCollection<OutT>> transform)

getPipelineOptions

public SamzaPipelineOptions getPipelineOptions()

getOutputStream

public <OutT> org.apache.samza.operators.OutputStream<OutT> getOutputStream(org.apache.samza.system.descriptors.OutputDescriptor<OutT,?> outputDescriptor)

getTable

public <K,V> org.apache.samza.table.Table<org.apache.samza.operators.KV<K,V>> getTable(org.apache.samza.table.descriptors.TableDescriptor<K,V,?> tableDesc)

getIdForPValue

public java.lang.String getIdForPValue(org.apache.beam.sdk.values.PValue pvalue)

getTransformFullName

public java.lang.String getTransformFullName()

getTransformId

public java.lang.String getTransformId()

getStoreIdGenerator

public StoreIdGenerator getStoreIdGenerator()