edu.umd.cloud9.io

Class FileMerger

java.lang.Object

org.apache.hadoop.conf.Configured

edu.umd.cloud9.io.FileMerger

All Implemented Interfaces:

org.apache.hadoop.conf.Configurable, org.apache.hadoop.util.Tool

public class FileMerger
extends org.apache.hadoop.conf.Configured
implements org.apache.hadoop.util.Tool

Field Summary

Fields

Modifier and Type	Field and Description
static int	DEFAULT_RANDOM_STRING_LENGTH
static boolean	DELETE_SOURCE
static String	DELETE_SOURCE_OPTION
static String	FILE_CONTENT_DELIMITER
static String	HELP_OPTION
static String	INPUT_OPTION
static String	INTEGER_INDICATOR
static boolean	LOCAL_MERGE
static String	LOCAL_MERGE_OPTION
static String	MAPPER_OPTION
static String	MERGE
static String	OUTPUT_OPTION
static String	PATH_INDICATOR
static Random	RANDOM_GENERATOR
static String	REDUCER_OPTION
static boolean	TEXT_FILE_INPUT
static String	TEXT_FILE_INPUT_FORMAT

Constructor Summary

Constructors

Constructor and Description
FileMerger()

Method Summary

Methods

Modifier and Type	Method and Description
static String	generateRandomString() Generate a random string of default length.
static String	generateRandomString(int length) Generate a random string of given length.
static void	main(String[] args)
static org.apache.hadoop.fs.Path	merge(org.apache.hadoop.conf.Configuration configuration, String inputPath, String outputPath, int mapperTasks, boolean textFileFormat, boolean deleteSource)
static org.apache.hadoop.fs.Path	mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, Class<? extends org.apache.hadoop.io.Writable> keyClass, Class<? extends org.apache.hadoop.io.Writable> valueClass, boolean deleteSource)
static org.apache.hadoop.fs.Path	mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, Class<? extends org.apache.hadoop.io.Writable> keyClass, Class<? extends org.apache.hadoop.io.Writable> valueClass, boolean deleteSource, boolean deleteDestinationFileIfExist)
static org.apache.hadoop.fs.Path	mergeTextFiles(org.apache.hadoop.conf.Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, boolean deleteSource) This method merges all files specified by the glob expression inputFiles
static org.apache.hadoop.fs.Path	mergeTextFiles(org.apache.hadoop.conf.Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, boolean deleteSource, boolean deleteDestinationFileIfExist)
int	run(String[] args)

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.hadoop.conf.Configurable

getConf, setConf

Field Detail

RANDOM_GENERATOR

public static final Random RANDOM_GENERATOR

DEFAULT_RANDOM_STRING_LENGTH

public static final int DEFAULT_RANDOM_STRING_LENGTH

See Also:

Constant Field Values

PATH_INDICATOR

public static final String PATH_INDICATOR

See Also:

Constant Field Values

INTEGER_INDICATOR

public static final String INTEGER_INDICATOR

See Also:

Constant Field Values

HELP_OPTION

public static final String HELP_OPTION

See Also:

Constant Field Values

INPUT_OPTION

public static final String INPUT_OPTION

See Also:

Constant Field Values

OUTPUT_OPTION

public static final String OUTPUT_OPTION

See Also:

Constant Field Values

MAPPER_OPTION

public static final String MAPPER_OPTION

See Also:

Constant Field Values

REDUCER_OPTION

public static final String REDUCER_OPTION

See Also:

Constant Field Values

MERGE

public static final String MERGE

See Also:

Constant Field Values

LOCAL_MERGE_OPTION

public static final String LOCAL_MERGE_OPTION

See Also:

Constant Field Values

LOCAL_MERGE

public static final boolean LOCAL_MERGE

See Also:

Constant Field Values

DELETE_SOURCE_OPTION

public static final String DELETE_SOURCE_OPTION

See Also:

Constant Field Values

DELETE_SOURCE

public static final boolean DELETE_SOURCE

See Also:

Constant Field Values

TEXT_FILE_INPUT_FORMAT

public static final String TEXT_FILE_INPUT_FORMAT

See Also:

Constant Field Values

TEXT_FILE_INPUT

public static final boolean TEXT_FILE_INPUT

See Also:

Constant Field Values

FILE_CONTENT_DELIMITER

public static final String FILE_CONTENT_DELIMITER

See Also:

Constant Field Values

Constructor Detail

FileMerger

public FileMerger()

Method Detail

generateRandomString

public static String generateRandomString(int length)

Generate a random string of given length.

generateRandomString

public static String generateRandomString()

Generate a random string of default length.

mergeTextFiles

public static org.apache.hadoop.fs.Path mergeTextFiles(org.apache.hadoop.conf.Configuration configuration,
                                       String inputFiles,
                                       String outputFile,
                                       int numberOfMappers,
                                       boolean deleteSource)
                                                throws IOException

This method merges all files specified by the glob expression inputFiles

Throws:

IOException

mergeTextFiles

public static org.apache.hadoop.fs.Path mergeTextFiles(org.apache.hadoop.conf.Configuration configuration,
                                       String inputFiles,
                                       String outputFile,
                                       int numberOfMappers,
                                       boolean deleteSource,
                                       boolean deleteDestinationFileIfExist)
                                                throws IOException

Throws:

IOException

mergeSequenceFiles

public static org.apache.hadoop.fs.Path mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration,
                                           String inputFiles,
                                           String outputFile,
                                           int numberOfMappers,
                                           Class<? extends org.apache.hadoop.io.Writable> keyClass,
                                           Class<? extends org.apache.hadoop.io.Writable> valueClass,
                                           boolean deleteSource)
                                                    throws IOException,
                                                           InstantiationException,
                                                           IllegalAccessException

Throws:

IOException

InstantiationException

IllegalAccessException

mergeSequenceFiles

public static org.apache.hadoop.fs.Path mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration,
                                           String inputFiles,
                                           String outputFile,
                                           int numberOfMappers,
                                           Class<? extends org.apache.hadoop.io.Writable> keyClass,
                                           Class<? extends org.apache.hadoop.io.Writable> valueClass,
                                           boolean deleteSource,
                                           boolean deleteDestinationFileIfExist)
                                                    throws IOException,
                                                           InstantiationException,
                                                           IllegalAccessException

Throws:

IOException

InstantiationException

IllegalAccessException

run

public int run(String[] args)
        throws IOException

Specified by:

run in interface org.apache.hadoop.util.Tool

Throws:

IOException

merge

public static org.apache.hadoop.fs.Path merge(org.apache.hadoop.conf.Configuration configuration,
                              String inputPath,
                              String outputPath,
                              int mapperTasks,
                              boolean textFileFormat,
                              boolean deleteSource)
                                       throws IOException,
                                              InstantiationException,
                                              IllegalAccessException

Throws:

IOException

InstantiationException

IllegalAccessException

main

public static void main(String[] args)
                 throws Exception

Throws:

Exception