org.apache.tez.runtime.library.common.sort.impl

Class TezMerger

java.lang.Object

org.apache.tez.runtime.library.common.sort.impl.TezMerger

@InterfaceAudience.Private
 @InterfaceStability.Unstable
public class TezMerger
extends Object

Merger is an utility class used by the Map and Reduce tasks for merging both their memory and disk segments

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	TezMerger.DiskSegment
static class	TezMerger.Segment

Constructor Summary

Constructors

Constructor and Description
TezMerger()

Method Summary

Modifier and Type	Method and Description
static TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, org.apache.hadoop.fs.Path[] inputs, boolean deleteInputs, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<TezMerger.Segment> segments, int mergeFactor, int inMemSegments, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<TezMerger.Segment> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, boolean considerFinalMergeForProgress, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<TezMerger.Segment> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, boolean considerFinalMergeForProgress, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase, boolean checkForSameKeys)
static <K,V> TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, List<TezMerger.Segment> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, List<TezMerger.Segment> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> void	writeFile(TezRawKeyValueIterator records, IFile.Writer writer, org.apache.hadoop.util.Progressable progressable, long recordsBeforeProgress)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TezMerger

public TezMerger()

Method Detail

merge

public static TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                           org.apache.hadoop.fs.FileSystem fs,
                                           Class keyClass,
                                           Class valueClass,
                                           org.apache.hadoop.io.compress.CompressionCodec codec,
                                           boolean ifileReadAhead,
                                           int ifileReadAheadLength,
                                           int ifileBufferSize,
                                           org.apache.hadoop.fs.Path[] inputs,
                                           boolean deleteInputs,
                                           int mergeFactor,
                                           org.apache.hadoop.fs.Path tmpDir,
                                           org.apache.hadoop.io.RawComparator comparator,
                                           org.apache.hadoop.util.Progressable reporter,
                                           org.apache.tez.common.counters.TezCounter readsCounter,
                                           org.apache.tez.common.counters.TezCounter writesCounter,
                                           org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                           org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException,
                                           InterruptedException

Throws:

IOException

InterruptedException

merge

public static TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                           org.apache.hadoop.fs.FileSystem fs,
                                           Class keyClass,
                                           Class valueClass,
                                           List<TezMerger.Segment> segments,
                                           int mergeFactor,
                                           org.apache.hadoop.fs.Path tmpDir,
                                           org.apache.hadoop.io.RawComparator comparator,
                                           org.apache.hadoop.util.Progressable reporter,
                                           org.apache.tez.common.counters.TezCounter readsCounter,
                                           org.apache.tez.common.counters.TezCounter writesCounter,
                                           org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                           org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException,
                                           InterruptedException

Throws:

IOException

InterruptedException

merge

public static <K,V> TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                                 org.apache.hadoop.fs.FileSystem fs,
                                                 Class keyClass,
                                                 Class valueClass,
                                                 List<TezMerger.Segment> segments,
                                                 int mergeFactor,
                                                 org.apache.hadoop.fs.Path tmpDir,
                                                 org.apache.hadoop.io.RawComparator comparator,
                                                 org.apache.hadoop.util.Progressable reporter,
                                                 boolean sortSegments,
                                                 org.apache.tez.common.counters.TezCounter readsCounter,
                                                 org.apache.tez.common.counters.TezCounter writesCounter,
                                                 org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                                 org.apache.hadoop.util.Progress mergePhase)
                                          throws IOException,
                                                 InterruptedException

Throws:

IOException

InterruptedException

merge

public static <K,V> TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                                 org.apache.hadoop.fs.FileSystem fs,
                                                 Class keyClass,
                                                 Class valueClass,
                                                 org.apache.hadoop.io.compress.CompressionCodec codec,
                                                 List<TezMerger.Segment> segments,
                                                 int mergeFactor,
                                                 org.apache.hadoop.fs.Path tmpDir,
                                                 org.apache.hadoop.io.RawComparator comparator,
                                                 org.apache.hadoop.util.Progressable reporter,
                                                 boolean sortSegments,
                                                 boolean considerFinalMergeForProgress,
                                                 org.apache.tez.common.counters.TezCounter readsCounter,
                                                 org.apache.tez.common.counters.TezCounter writesCounter,
                                                 org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                                 org.apache.hadoop.util.Progress mergePhase,
                                                 boolean checkForSameKeys)
                                          throws IOException,
                                                 InterruptedException

Throws:

IOException

InterruptedException

merge

public static <K,V> TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                                 org.apache.hadoop.fs.FileSystem fs,
                                                 Class keyClass,
                                                 Class valueClass,
                                                 org.apache.hadoop.io.compress.CompressionCodec codec,
                                                 List<TezMerger.Segment> segments,
                                                 int mergeFactor,
                                                 org.apache.hadoop.fs.Path tmpDir,
                                                 org.apache.hadoop.io.RawComparator comparator,
                                                 org.apache.hadoop.util.Progressable reporter,
                                                 boolean sortSegments,
                                                 boolean considerFinalMergeForProgress,
                                                 org.apache.tez.common.counters.TezCounter readsCounter,
                                                 org.apache.tez.common.counters.TezCounter writesCounter,
                                                 org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                                 org.apache.hadoop.util.Progress mergePhase)
                                          throws IOException,
                                                 InterruptedException

Throws:

IOException

InterruptedException

merge

public static <K,V> TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                                 org.apache.hadoop.fs.FileSystem fs,
                                                 Class keyClass,
                                                 Class valueClass,
                                                 org.apache.hadoop.io.compress.CompressionCodec codec,
                                                 List<TezMerger.Segment> segments,
                                                 int mergeFactor,
                                                 int inMemSegments,
                                                 org.apache.hadoop.fs.Path tmpDir,
                                                 org.apache.hadoop.io.RawComparator comparator,
                                                 org.apache.hadoop.util.Progressable reporter,
                                                 boolean sortSegments,
                                                 org.apache.tez.common.counters.TezCounter readsCounter,
                                                 org.apache.tez.common.counters.TezCounter writesCounter,
                                                 org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                                 org.apache.hadoop.util.Progress mergePhase)
                                          throws IOException,
                                                 InterruptedException

Throws:

IOException

InterruptedException

writeFile

public static <K,V> void writeFile(TezRawKeyValueIterator records,
                                   IFile.Writer writer,
                                   org.apache.hadoop.util.Progressable progressable,
                                   long recordsBeforeProgress)
                            throws IOException,
                                   InterruptedException

Throws:

IOException

InterruptedException