public class FileMerger
extends org.apache.hadoop.conf.Configured
implements org.apache.hadoop.util.Tool
| Modifier and Type | Field and Description |
|---|---|
static int |
DEFAULT_RANDOM_STRING_LENGTH |
static boolean |
DELETE_SOURCE |
static String |
DELETE_SOURCE_OPTION |
static String |
FILE_CONTENT_DELIMITER |
static String |
HELP_OPTION |
static String |
INPUT_OPTION |
static String |
INTEGER_INDICATOR |
static boolean |
LOCAL_MERGE |
static String |
LOCAL_MERGE_OPTION |
static String |
MAPPER_OPTION |
static String |
MERGE |
static String |
OUTPUT_OPTION |
static String |
PATH_INDICATOR |
static Random |
RANDOM_GENERATOR |
static String |
REDUCER_OPTION |
static boolean |
TEXT_FILE_INPUT |
static String |
TEXT_FILE_INPUT_FORMAT |
| Constructor and Description |
|---|
FileMerger() |
| Modifier and Type | Method and Description |
|---|---|
static String |
generateRandomString()
Generate a random string of default length.
|
static String |
generateRandomString(int length)
Generate a random string of given length.
|
static void |
main(String[] args) |
static org.apache.hadoop.fs.Path |
merge(org.apache.hadoop.conf.Configuration configuration,
String inputPath,
String outputPath,
int mapperTasks,
boolean textFileFormat,
boolean deleteSource) |
static org.apache.hadoop.fs.Path |
mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
Class<? extends org.apache.hadoop.io.Writable> keyClass,
Class<? extends org.apache.hadoop.io.Writable> valueClass,
boolean deleteSource) |
static org.apache.hadoop.fs.Path |
mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
Class<? extends org.apache.hadoop.io.Writable> keyClass,
Class<? extends org.apache.hadoop.io.Writable> valueClass,
boolean deleteSource,
boolean deleteDestinationFileIfExist) |
static org.apache.hadoop.fs.Path |
mergeTextFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
boolean deleteSource)
This method merges all files specified by the glob expression
inputFiles |
static org.apache.hadoop.fs.Path |
mergeTextFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
boolean deleteSource,
boolean deleteDestinationFileIfExist) |
int |
run(String[] args) |
public static final Random RANDOM_GENERATOR
public static final int DEFAULT_RANDOM_STRING_LENGTH
public static final String PATH_INDICATOR
public static final String INTEGER_INDICATOR
public static final String HELP_OPTION
public static final String INPUT_OPTION
public static final String OUTPUT_OPTION
public static final String MAPPER_OPTION
public static final String REDUCER_OPTION
public static final String MERGE
public static final String LOCAL_MERGE_OPTION
public static final boolean LOCAL_MERGE
public static final String DELETE_SOURCE_OPTION
public static final boolean DELETE_SOURCE
public static final String TEXT_FILE_INPUT_FORMAT
public static final boolean TEXT_FILE_INPUT
public static final String FILE_CONTENT_DELIMITER
public static String generateRandomString(int length)
public static String generateRandomString()
public static org.apache.hadoop.fs.Path mergeTextFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
boolean deleteSource)
throws IOException
inputFilesIOExceptionpublic static org.apache.hadoop.fs.Path mergeTextFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
boolean deleteSource,
boolean deleteDestinationFileIfExist)
throws IOException
IOExceptionpublic static org.apache.hadoop.fs.Path mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
Class<? extends org.apache.hadoop.io.Writable> keyClass,
Class<? extends org.apache.hadoop.io.Writable> valueClass,
boolean deleteSource)
throws IOException,
InstantiationException,
IllegalAccessException
public static org.apache.hadoop.fs.Path mergeSequenceFiles(org.apache.hadoop.conf.Configuration configuration,
String inputFiles,
String outputFile,
int numberOfMappers,
Class<? extends org.apache.hadoop.io.Writable> keyClass,
Class<? extends org.apache.hadoop.io.Writable> valueClass,
boolean deleteSource,
boolean deleteDestinationFileIfExist)
throws IOException,
InstantiationException,
IllegalAccessException
public int run(String[] args) throws IOException
run in interface org.apache.hadoop.util.ToolIOExceptionpublic static org.apache.hadoop.fs.Path merge(org.apache.hadoop.conf.Configuration configuration,
String inputPath,
String outputPath,
int mapperTasks,
boolean textFileFormat,
boolean deleteSource)
throws IOException,
InstantiationException,
IllegalAccessException
Copyright © 2015. All rights reserved.