org.apache.tez.runtime.library.common.shuffle

Class ShuffleUtils

java.lang.Object

org.apache.tez.runtime.library.common.shuffle.ShuffleUtils

public class ShuffleUtils
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	ShuffleUtils.FetchStatsLogger

Constructor Summary

Constructors

Constructor and Description
ShuffleUtils()

Method Summary

Modifier and Type	Method and Description
static StringBuilder	constructBaseURIForShuffleHandler(String host, int port, int partition, int partitionCount, String appId, int dagIdentifier, boolean sslShuffle)
static URL	constructInputURL(String baseURI, Collection<InputAttemptIdentifier> inputs, boolean keepAlive)
static ByteBuffer	convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken)
static int	deserializeShuffleProviderMetaData(ByteBuffer meta)
static void	generateEventOnSpill(List<org.apache.tez.runtime.api.Event> eventList, boolean finalMergeEnabled, boolean isLastEvent, org.apache.tez.runtime.api.OutputContext context, int spillId, TezSpillRecord spillRecord, int numPhysicalOutputs, boolean sendEmptyPartitionDetails, String pathComponent, long[] partitionStats, boolean reportDetailedPartitionStats, String auxiliaryService, Deflater deflater) Generate events when spill happens
static void	generateEventsForNonStartedOutput(List<org.apache.tez.runtime.api.Event> eventList, int numPhysicalOutputs, org.apache.tez.runtime.api.OutputContext context, boolean generateVmEvent, boolean isCompositeEvent, Deflater deflater) Generate events for outputs which have not been started.
static org.apache.tez.runtime.api.events.VertexManagerEvent	generateVMEvent(org.apache.tez.runtime.api.OutputContext context, long[] sizePerPartition, boolean reportDetailedPartitionStats, Deflater deflater)
static ShuffleUserPayloads.DetailedPartitionStatsProto	getDetailedPartitionStatsForPhysicalOutput(long[] sizes) Detailed partition stats
static BaseHttpConnection	getHttpConnection(boolean asyncHttp, URL url, HttpConnectionParams params, String logIdentifier, org.apache.tez.common.security.JobTokenSecretManager jobTokenSecretManager)
static HttpConnectionParams	getHttpConnectionParams(org.apache.hadoop.conf.Configuration conf) Build HttpConnectionParams from configuration
static SecretKey	getJobTokenSecretFromTokenBytes(ByteBuffer meta)
static org.roaringbitmap.RoaringBitmap	getPartitionStatsForPhysicalOutput(long[] sizes) Data size for the destinations
static void	ioCleanup(Closeable... closeables)
static boolean	isTezShuffleHandler(org.apache.hadoop.conf.Configuration config)
static void	shuffleToDisk(OutputStream output, String hostIdentifier, InputStream input, long compressedLength, long decompressedLength, org.slf4j.Logger LOG, InputAttemptIdentifier identifier, boolean ifileReadAhead, int ifileReadAheadLength, boolean verifyChecksum)
static void	shuffleToMemory(byte[] shuffleData, InputStream input, int decompressedLength, int compressedLength, org.apache.hadoop.io.compress.CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, org.slf4j.Logger LOG, InputAttemptIdentifier identifier)
static String	stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

ShuffleUtils

public ShuffleUtils()

Method Detail

getJobTokenSecretFromTokenBytes

public static SecretKey getJobTokenSecretFromTokenBytes(ByteBuffer meta)
                                                 throws IOException

Throws:

IOException

convertJobTokenToBytes

public static ByteBuffer convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken)
                                         throws IOException

Throws:

IOException

deserializeShuffleProviderMetaData

public static int deserializeShuffleProviderMetaData(ByteBuffer meta)
                                              throws IOException

Throws:

IOException

shuffleToMemory

public static void shuffleToMemory(byte[] shuffleData,
                                   InputStream input,
                                   int decompressedLength,
                                   int compressedLength,
                                   org.apache.hadoop.io.compress.CompressionCodec codec,
                                   boolean ifileReadAhead,
                                   int ifileReadAheadLength,
                                   org.slf4j.Logger LOG,
                                   InputAttemptIdentifier identifier)
                            throws IOException

Throws:

IOException

shuffleToDisk

public static void shuffleToDisk(OutputStream output,
                                 String hostIdentifier,
                                 InputStream input,
                                 long compressedLength,
                                 long decompressedLength,
                                 org.slf4j.Logger LOG,
                                 InputAttemptIdentifier identifier,
                                 boolean ifileReadAhead,
                                 int ifileReadAheadLength,
                                 boolean verifyChecksum)
                          throws IOException

Throws:

IOException

ioCleanup

public static void ioCleanup(Closeable... closeables)

constructBaseURIForShuffleHandler

public static StringBuilder constructBaseURIForShuffleHandler(String host,
                                                              int port,
                                                              int partition,
                                                              int partitionCount,
                                                              String appId,
                                                              int dagIdentifier,
                                                              boolean sslShuffle)

constructInputURL

public static URL constructInputURL(String baseURI,
                                    Collection<InputAttemptIdentifier> inputs,
                                    boolean keepAlive)
                             throws MalformedURLException

Throws:

MalformedURLException

getHttpConnection

public static BaseHttpConnection getHttpConnection(boolean asyncHttp,
                                                   URL url,
                                                   HttpConnectionParams params,
                                                   String logIdentifier,
                                                   org.apache.tez.common.security.JobTokenSecretManager jobTokenSecretManager)
                                            throws IOException

Throws:

IOException

stringify

public static String stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto)

generateEventsForNonStartedOutput

public static void generateEventsForNonStartedOutput(List<org.apache.tez.runtime.api.Event> eventList,
                                                     int numPhysicalOutputs,
                                                     org.apache.tez.runtime.api.OutputContext context,
                                                     boolean generateVmEvent,
                                                     boolean isCompositeEvent,
                                                     Deflater deflater)
                                              throws IOException

Generate events for outputs which have not been started.

Parameters:

eventList -

numPhysicalOutputs -

context -

generateVmEvent - whether to generate a vm event or not

isCompositeEvent - whether to generate a CompositeDataMovementEvent or a DataMovementEvent

deflater -

Throws:

IOException

generateEventOnSpill

public static void generateEventOnSpill(List<org.apache.tez.runtime.api.Event> eventList,
                                        boolean finalMergeEnabled,
                                        boolean isLastEvent,
                                        org.apache.tez.runtime.api.OutputContext context,
                                        int spillId,
                                        TezSpillRecord spillRecord,
                                        int numPhysicalOutputs,
                                        boolean sendEmptyPartitionDetails,
                                        String pathComponent,
                                        @Nullable
                                        long[] partitionStats,
                                        boolean reportDetailedPartitionStats,
                                        String auxiliaryService,
                                        Deflater deflater)
                                 throws IOException

Generate events when spill happens

Parameters:

eventList - events would be added to this list

finalMergeEnabled -

isLastEvent -

context -

spillId -

spillRecord -

numPhysicalOutputs -

pathComponent -

partitionStats -

auxiliaryService -

Throws:

IOException

generateVMEvent

public static org.apache.tez.runtime.api.events.VertexManagerEvent generateVMEvent(org.apache.tez.runtime.api.OutputContext context,
                                                                                   long[] sizePerPartition,
                                                                                   boolean reportDetailedPartitionStats,
                                                                                   Deflater deflater)
                                                                            throws IOException

Throws:

IOException

getPartitionStatsForPhysicalOutput

public static org.roaringbitmap.RoaringBitmap getPartitionStatsForPhysicalOutput(long[] sizes)

Data size for the destinations

Parameters:

sizes - for physical outputs

getDetailedPartitionStatsForPhysicalOutput

public static ShuffleUserPayloads.DetailedPartitionStatsProto getDetailedPartitionStatsForPhysicalOutput(long[] sizes)

Detailed partition stats

Parameters:

sizes - actual partition sizes

getHttpConnectionParams

public static HttpConnectionParams getHttpConnectionParams(org.apache.hadoop.conf.Configuration conf)

Build HttpConnectionParams from configuration

Parameters:

conf -

Returns:

HttpConnectionParams

isTezShuffleHandler

public static boolean isTezShuffleHandler(org.apache.hadoop.conf.Configuration config)