org.apache.carbondata.hadoop.api

Class CarbonTableOutputFormat

java.lang.Object

org.apache.hadoop.mapreduce.OutputFormat<K,V>

org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<org.apache.hadoop.io.NullWritable,ObjectArrayWritable>

org.apache.carbondata.hadoop.api.CarbonTableOutputFormat

public class CarbonTableOutputFormat
extends org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<org.apache.hadoop.io.NullWritable,ObjectArrayWritable>

This is table level output format which writes the data to store in new segment. Each load creates new segment folder and manages the folder through tablestatus file. It also generate and writes dictionary data during load only if dictionary server is configured.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	CarbonTableOutputFormat.CarbonMultiRecordWriter
static class	CarbonTableOutputFormat.CarbonRecordWriter

Nested classes/interfaces inherited from class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat

org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter

Field Summary

Fields

Modifier and Type	Field and Description
static String	BAD_RECORD_PATH
static String	BAD_RECORDS_LOGGER_ACTION
static String	BAD_RECORDS_LOGGER_ENABLE
static String	BATCH_SORT_SIZE_INMB
static String	COMPLEX_DELIMITERS
static String	DATE_FORMAT
static String	DICTIONARY_SERVER_HOST
static String	DICTIONARY_SERVER_PORT
static String	GLOBAL_SORT_PARTITIONS
static String	IS_EMPTY_DATA_BAD_RECORD
static String	IS_ONE_PASS_LOAD
static String	OPERATION_CONTEXT It is used only to fire events in case of any child tables to be loaded.
static String	SEGMENTS_TO_BE_DELETED During update query we first delete the old data and then add updated data to new segment, so sometimes there is a chance that complete segments needs to removed during deletion.
static String	SERIALIZATION_NULL_FORMAT
static String	SKIP_EMPTY_LINE
static String	SORT_SCOPE
static String	TIMESTAMP_FORMAT
static String	UPADTE_TIMESTAMP Set the update timestamp if user sets in case of update query.

Fields inherited from class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat

BASE_OUTPUT_NAME, COMPRESS, COMPRESS_CODEC, COMPRESS_TYPE, OUTDIR, PART

Constructor Summary

Constructors

Constructor and Description
CarbonTableOutputFormat()

Method Summary

Modifier and Type	Method and Description
static org.apache.carbondata.core.metadata.schema.table.CarbonTable	getCarbonTable(org.apache.hadoop.conf.Configuration configuration)
static String	getDatabaseName(org.apache.hadoop.conf.Configuration configuration)
static CarbonLoadModel	getLoadModel(org.apache.hadoop.conf.Configuration conf)
org.apache.hadoop.mapreduce.OutputCommitter	getOutputCommitter(org.apache.hadoop.mapreduce.TaskAttemptContext context)
org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.io.NullWritable,ObjectArrayWritable>	getRecordWriter(org.apache.hadoop.mapreduce.TaskAttemptContext taskAttemptContext)
static String	getTableName(org.apache.hadoop.conf.Configuration configuration)
static String	getTablePath(org.apache.hadoop.conf.Configuration configuration)
static boolean	isOverwriteSet(org.apache.hadoop.conf.Configuration configuration)
static void	setCarbonTable(org.apache.hadoop.conf.Configuration configuration, org.apache.carbondata.core.metadata.schema.table.CarbonTable carbonTable)
static void	setDatabaseName(org.apache.hadoop.conf.Configuration configuration, String databaseName)
static void	setInputSchema(org.apache.hadoop.conf.Configuration configuration, org.apache.carbondata.core.metadata.datatype.StructType inputSchema)
static void	setLoadModel(org.apache.hadoop.conf.Configuration configuration, CarbonLoadModel loadModel)
static void	setOverwrite(org.apache.hadoop.conf.Configuration configuration, boolean overwrite)
static void	setTableName(org.apache.hadoop.conf.Configuration configuration, String tableName)
static void	setTablePath(org.apache.hadoop.conf.Configuration configuration, String tablePath)
static void	setTempStoreLocations(org.apache.hadoop.conf.Configuration configuration, String[] tempLocations)

Methods inherited from class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat

checkOutputSpecs, getCompressOutput, getDefaultWorkFile, getOutputCompressorClass, getOutputName, getOutputPath, getPathForWorkFile, getUniqueFile, getWorkOutputPath, setCompressOutput, setOutputCompressorClass, setOutputName, setOutputPath

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

COMPLEX_DELIMITERS

public static final String COMPLEX_DELIMITERS

See Also:

Constant Field Values

SERIALIZATION_NULL_FORMAT

public static final String SERIALIZATION_NULL_FORMAT

See Also:

Constant Field Values

BAD_RECORDS_LOGGER_ENABLE

public static final String BAD_RECORDS_LOGGER_ENABLE

See Also:

Constant Field Values

BAD_RECORDS_LOGGER_ACTION

public static final String BAD_RECORDS_LOGGER_ACTION

See Also:

Constant Field Values

IS_EMPTY_DATA_BAD_RECORD

public static final String IS_EMPTY_DATA_BAD_RECORD

See Also:

Constant Field Values

SKIP_EMPTY_LINE

public static final String SKIP_EMPTY_LINE

See Also:

Constant Field Values

SORT_SCOPE

public static final String SORT_SCOPE

See Also:

Constant Field Values

BATCH_SORT_SIZE_INMB

public static final String BATCH_SORT_SIZE_INMB

See Also:

Constant Field Values

GLOBAL_SORT_PARTITIONS

public static final String GLOBAL_SORT_PARTITIONS

See Also:

Constant Field Values

BAD_RECORD_PATH

public static final String BAD_RECORD_PATH

See Also:

Constant Field Values

DATE_FORMAT

public static final String DATE_FORMAT

See Also:

Constant Field Values

TIMESTAMP_FORMAT

public static final String TIMESTAMP_FORMAT

See Also:

Constant Field Values

IS_ONE_PASS_LOAD

public static final String IS_ONE_PASS_LOAD

See Also:

Constant Field Values

DICTIONARY_SERVER_HOST

public static final String DICTIONARY_SERVER_HOST

See Also:

Constant Field Values

DICTIONARY_SERVER_PORT

public static final String DICTIONARY_SERVER_PORT

See Also:

Constant Field Values

UPADTE_TIMESTAMP

public static final String UPADTE_TIMESTAMP

Set the update timestamp if user sets in case of update query. It needs to be updated in load status update time

See Also:

Constant Field Values

SEGMENTS_TO_BE_DELETED

public static final String SEGMENTS_TO_BE_DELETED

During update query we first delete the old data and then add updated data to new segment, so sometimes there is a chance that complete segments needs to removed during deletion. We should do 'Mark for delete' for those segments during table status update.

See Also:

Constant Field Values

OPERATION_CONTEXT

public static final String OPERATION_CONTEXT

It is used only to fire events in case of any child tables to be loaded.

See Also:

Constant Field Values

Constructor Detail

CarbonTableOutputFormat

public CarbonTableOutputFormat()

Method Detail

setDatabaseName

public static void setDatabaseName(org.apache.hadoop.conf.Configuration configuration,
                                   String databaseName)

getDatabaseName

public static String getDatabaseName(org.apache.hadoop.conf.Configuration configuration)

setTableName

public static void setTableName(org.apache.hadoop.conf.Configuration configuration,
                                String tableName)

getTableName

public static String getTableName(org.apache.hadoop.conf.Configuration configuration)

setTablePath

public static void setTablePath(org.apache.hadoop.conf.Configuration configuration,
                                String tablePath)

getTablePath

public static String getTablePath(org.apache.hadoop.conf.Configuration configuration)

setCarbonTable

public static void setCarbonTable(org.apache.hadoop.conf.Configuration configuration,
                                  org.apache.carbondata.core.metadata.schema.table.CarbonTable carbonTable)
                           throws IOException

Throws:

IOException

getCarbonTable

public static org.apache.carbondata.core.metadata.schema.table.CarbonTable getCarbonTable(org.apache.hadoop.conf.Configuration configuration)
                                                                                   throws IOException

Throws:

IOException

setLoadModel

public static void setLoadModel(org.apache.hadoop.conf.Configuration configuration,
                                CarbonLoadModel loadModel)
                         throws IOException

Throws:

IOException

setInputSchema

public static void setInputSchema(org.apache.hadoop.conf.Configuration configuration,
                                  org.apache.carbondata.core.metadata.datatype.StructType inputSchema)
                           throws IOException

Throws:

IOException

isOverwriteSet

public static boolean isOverwriteSet(org.apache.hadoop.conf.Configuration configuration)

setOverwrite

public static void setOverwrite(org.apache.hadoop.conf.Configuration configuration,
                                boolean overwrite)

setTempStoreLocations

public static void setTempStoreLocations(org.apache.hadoop.conf.Configuration configuration,
                                         String[] tempLocations)
                                  throws IOException

Throws:

IOException

getOutputCommitter

public org.apache.hadoop.mapreduce.OutputCommitter getOutputCommitter(org.apache.hadoop.mapreduce.TaskAttemptContext context)
                                                               throws IOException

Overrides:

getOutputCommitter in class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<org.apache.hadoop.io.NullWritable,ObjectArrayWritable>

Throws:

IOException

getRecordWriter

public org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.io.NullWritable,ObjectArrayWritable> getRecordWriter(org.apache.hadoop.mapreduce.TaskAttemptContext taskAttemptContext)
                                                                                                                throws IOException

Specified by:

getRecordWriter in class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<org.apache.hadoop.io.NullWritable,ObjectArrayWritable>

Throws:

IOException

getLoadModel

public static CarbonLoadModel getLoadModel(org.apache.hadoop.conf.Configuration conf)
                                    throws IOException

Throws:

IOException