org.apache.druid.indexer

Class HadoopTuningConfig

java.lang.Object

org.apache.druid.indexer.HadoopTuningConfig

All Implemented Interfaces:

TuningConfig

public class HadoopTuningConfig
extends Object
implements TuningConfig

Field Summary

Fields

Modifier and Type	Field and Description
static int	DEFAULT_DETERMINE_PARTITIONS_SAMPLING_FACTOR

Fields inherited from interface org.apache.druid.segment.indexing.TuningConfig

DEFAULT_APPENDABLE_INDEX, DEFAULT_AWAIT_SEGMENT_AVAILABILITY_TIMEOUT_MILLIS, DEFAULT_LOG_PARSE_EXCEPTIONS, DEFAULT_MAX_PARSE_EXCEPTIONS, DEFAULT_MAX_ROWS_IN_MEMORY_BATCH, DEFAULT_MAX_ROWS_IN_MEMORY_REALTIME, DEFAULT_MAX_SAVED_PARSE_EXCEPTIONS, DEFAULT_SKIP_BYTES_IN_MEMORY_OVERHEAD_CHECK

Constructor Summary

Constructors

Constructor and Description

HadoopTuningConfig(String workingPath, String version, DimensionBasedPartitionsSpec partitionsSpec, Map<Long,List<HadoopyShardSpec>> shardSpecs, IndexSpec indexSpec, IndexSpec indexSpecForIntermediatePersists, AppendableIndexSpec appendableIndexSpec, Integer maxRowsInMemory, Long maxBytesInMemory, Boolean useMaxMemoryEstimates, boolean leaveIntermediate, Boolean cleanupOnFailure, boolean overwriteFiles, Boolean ignoreInvalidRows, Map<String,String> jobProperties, boolean combineText, Boolean useCombiner, Integer maxRowsInMemoryCOMPAT, Integer numBackgroundPersistThreads, boolean forceExtendableShardSpecs, boolean useExplicitVersion, List<String> allowedHadoopPrefix, Boolean logParseExceptions, Integer maxParseExceptions, Boolean useYarnRMJobStatusFallback, Long awaitSegmentAvailabilityTimeoutMillis, Integer determinePartitionsSamplingFactor)

Method Summary

Modifier and Type	Method and Description
AppendableIndexSpec	getAppendableIndexSpec()
long	getAwaitSegmentAvailabilityTimeoutMillis()
int	getDeterminePartitionsSamplingFactor()
IndexSpec	getIndexSpec()
IndexSpec	getIndexSpecForIntermediatePersists()
Map<String,String>	getJobProperties()
long	getMaxBytesInMemory()
int	getMaxParseExceptions()
int	getMaxRowsInMemory()
int	getNumBackgroundPersistThreads()
DimensionBasedPartitionsSpec	getPartitionsSpec()
Map<Long,List<HadoopyShardSpec>>	getShardSpecs()
boolean	getUseCombiner()
List<String>	getUserAllowedHadoopPrefix()
String	getVersion()
String	getWorkingPath()
Boolean	isCleanupOnFailure()
boolean	isCombineText()
boolean	isForceExtendableShardSpecs()
Boolean	isIgnoreInvalidRows()
boolean	isLeaveIntermediate()
boolean	isLogParseExceptions()
boolean	isOverwriteFiles()
boolean	isUseExplicitVersion()
boolean	isUseMaxMemoryEstimates()
boolean	isUseYarnRMJobStatusFallback()
static HadoopTuningConfig	makeDefaultTuningConfig()
HadoopTuningConfig	withShardSpecs(Map<Long,List<HadoopyShardSpec>> specs)
HadoopTuningConfig	withVersion(String ver)
HadoopTuningConfig	withWorkingPath(String path)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.druid.segment.indexing.TuningConfig

getMaxBytesInMemoryOrDefault

Field Detail

DEFAULT_DETERMINE_PARTITIONS_SAMPLING_FACTOR

public static final int DEFAULT_DETERMINE_PARTITIONS_SAMPLING_FACTOR

See Also:

Constant Field Values

Constructor Detail

HadoopTuningConfig

public HadoopTuningConfig(@Nullable
                          String workingPath,
                          @Nullable
                          String version,
                          @Nullable
                          DimensionBasedPartitionsSpec partitionsSpec,
                          @Nullable
                          Map<Long,List<HadoopyShardSpec>> shardSpecs,
                          @Nullable
                          IndexSpec indexSpec,
                          @Nullable
                          IndexSpec indexSpecForIntermediatePersists,
                          @Nullable
                          AppendableIndexSpec appendableIndexSpec,
                          @Nullable
                          Integer maxRowsInMemory,
                          @Nullable
                          Long maxBytesInMemory,
                          @Nullable
                          Boolean useMaxMemoryEstimates,
                          boolean leaveIntermediate,
                          @Nullable
                          Boolean cleanupOnFailure,
                          boolean overwriteFiles,
                          @Deprecated @Nullable
                          Boolean ignoreInvalidRows,
                          @Nullable
                          Map<String,String> jobProperties,
                          boolean combineText,
                          @Nullable
                          Boolean useCombiner,
                          @Nullable
                          Integer maxRowsInMemoryCOMPAT,
                          @Nullable
                          Integer numBackgroundPersistThreads,
                          boolean forceExtendableShardSpecs,
                          boolean useExplicitVersion,
                          @Nullable
                          List<String> allowedHadoopPrefix,
                          @Nullable
                          Boolean logParseExceptions,
                          @Nullable
                          Integer maxParseExceptions,
                          @Nullable
                          Boolean useYarnRMJobStatusFallback,
                          @Nullable
                          Long awaitSegmentAvailabilityTimeoutMillis,
                          @Nullable
                          Integer determinePartitionsSamplingFactor)

Method Detail

makeDefaultTuningConfig

public static HadoopTuningConfig makeDefaultTuningConfig()

getWorkingPath

@Nullable
public String getWorkingPath()

getVersion

public String getVersion()

getPartitionsSpec

public DimensionBasedPartitionsSpec getPartitionsSpec()

Specified by:

getPartitionsSpec in interface TuningConfig

getShardSpecs

public Map<Long,List<HadoopyShardSpec>> getShardSpecs()

getIndexSpec

public IndexSpec getIndexSpec()

Specified by:

getIndexSpec in interface TuningConfig

getIndexSpecForIntermediatePersists

public IndexSpec getIndexSpecForIntermediatePersists()

Specified by:

getIndexSpecForIntermediatePersists in interface TuningConfig

getAppendableIndexSpec

public AppendableIndexSpec getAppendableIndexSpec()

Specified by:

getAppendableIndexSpec in interface TuningConfig

getMaxRowsInMemory

public int getMaxRowsInMemory()

Specified by:

getMaxRowsInMemory in interface TuningConfig

getMaxBytesInMemory

public long getMaxBytesInMemory()

Specified by:

getMaxBytesInMemory in interface TuningConfig

isUseMaxMemoryEstimates

public boolean isUseMaxMemoryEstimates()

isLeaveIntermediate

public boolean isLeaveIntermediate()

isCleanupOnFailure

public Boolean isCleanupOnFailure()

isOverwriteFiles

public boolean isOverwriteFiles()

isIgnoreInvalidRows

public Boolean isIgnoreInvalidRows()

getJobProperties

public Map<String,String> getJobProperties()

isCombineText

public boolean isCombineText()

getUseCombiner

public boolean getUseCombiner()

getNumBackgroundPersistThreads

public int getNumBackgroundPersistThreads()

isForceExtendableShardSpecs

public boolean isForceExtendableShardSpecs()

isUseExplicitVersion

public boolean isUseExplicitVersion()

getUserAllowedHadoopPrefix

public List<String> getUserAllowedHadoopPrefix()

isLogParseExceptions

public boolean isLogParseExceptions()

getMaxParseExceptions

public int getMaxParseExceptions()

isUseYarnRMJobStatusFallback

public boolean isUseYarnRMJobStatusFallback()

getAwaitSegmentAvailabilityTimeoutMillis

public long getAwaitSegmentAvailabilityTimeoutMillis()

getDeterminePartitionsSamplingFactor

public int getDeterminePartitionsSamplingFactor()

withWorkingPath

public HadoopTuningConfig withWorkingPath(String path)

withVersion

public HadoopTuningConfig withVersion(String ver)

withShardSpecs

public HadoopTuningConfig withShardSpecs(Map<Long,List<HadoopyShardSpec>> specs)