class MultiFileCloudOrcPartitionReader extends MultiFileCloudPartitionReaderBase with MultiFileReaderFunctions with OrcPartitionReaderBase
A PartitionReader that can read multiple ORC files in parallel. This is most efficient running in a cloud environment where the I/O of reading is slow.
Efficiently reading a ORC split on the GPU requires re-constructing the ORC file in memory that contains just the Stripes that are needed. This avoids sending unnecessary data to the GPU and saves GPU memory.
- Alphabetic
- By Inheritance
- MultiFileCloudOrcPartitionReader
- OrcPartitionReaderBase
- OrcCommonFunctions
- OrcCodecWritingHelper
- MultiFileReaderFunctions
- MultiFileCloudPartitionReaderBase
- FilePartitionReaderBase
- ScanWithMetrics
- Logging
- PartitionReader
- Closeable
- AutoCloseable
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
MultiFileCloudOrcPartitionReader(conf: Configuration, files: Array[PartitionedFile], dataSchema: StructType, readDataSchema: StructType, partitionSchema: StructType, maxReadBatchSizeRows: Integer, maxReadBatchSizeBytes: Long, targetBatchSizeBytes: Long, maxGpuColumnSizeBytes: Long, useChunkedReader: Boolean, maxChunkedReaderMemoryUsageSizeBytes: Long, numThreads: Int, maxNumFileProcessed: Int, debugDumpPrefix: Option[String], debugDumpAlways: Boolean, filters: Array[Filter], filterHandler: GpuOrcFileFilterHandler, execMetrics: Map[String, GpuMetric], ignoreMissingFiles: Boolean, ignoreCorruptFiles: Boolean, queryUsesInputFile: Boolean, keepReadsInOrder: Boolean, combineConf: CombineConf)
- conf
the Hadoop configuration
- files
the partitioned files to read
- dataSchema
schema of the data
- readDataSchema
the Spark schema describing what will be read
- partitionSchema
Schema of partitions.
- maxReadBatchSizeRows
soft limit on the maximum number of rows the reader reads per batch
- maxReadBatchSizeBytes
soft limit on the maximum number of bytes the reader reads per batch
- targetBatchSizeBytes
the target size of a batch
- maxGpuColumnSizeBytes
maximum number of bytes for a GPU column
- useChunkedReader
whether to read Parquet by chunks or read all at once
- maxChunkedReaderMemoryUsageSizeBytes
soft limit on the number of bytes of internal memory usage that the reader will use
- numThreads
the size of the threadpool
- maxNumFileProcessed
threshold to control the maximum file number to be submitted to threadpool
- debugDumpPrefix
a path prefix to use for dumping the fabricated ORC data or null
- debugDumpAlways
whether to always debug dump or only on errors
- filters
filters passed into the filterHandler
- filterHandler
used to filter the ORC stripes
- execMetrics
the metrics
- ignoreMissingFiles
Whether to ignore missing files
- ignoreCorruptFiles
Whether to ignore corrupt files
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
var
batchIter: Iterator[ColumnarBatch]
- Attributes
- protected
- Definition Classes
- FilePartitionReaderBase
-
def
buildReaderSchema(updatedSchema: TypeDescription, requestedMapping: Option[Array[Int]]): TypeDescription
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
def
buildReaderSchema(ctx: OrcPartitionReaderContext): TypeDescription
Get the ORC schema corresponding to the file being constructed for the GPU
Get the ORC schema corresponding to the file being constructed for the GPU
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
final
def
calculateFileTailSize(ctx: OrcPartitionReaderContext, footerStartOffset: Long, stripes: Seq[OrcOutputStripe]): Long
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
def
canUseCombine: Boolean
- Definition Classes
- MultiFileCloudOrcPartitionReader → MultiFileCloudPartitionReaderBase
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
close(): Unit
- Definition Classes
- MultiFileCloudPartitionReaderBase → FilePartitionReaderBase → Closeable → AutoCloseable
-
def
combineHMBs(buffers: Array[HostMemoryBuffersWithMetaDataBase]): HostMemoryBuffersWithMetaDataBase
- Definition Classes
- MultiFileCloudOrcPartitionReader → MultiFileCloudPartitionReaderBase
-
var
combineLeftOverFiles: Option[Array[HostMemoryBuffersWithMetaDataBase]]
- Attributes
- protected
- Definition Classes
- MultiFileCloudPartitionReaderBase
-
val
conf: Configuration
- Definition Classes
- MultiFileCloudOrcPartitionReader → OrcCommonFunctions
-
def
copyStripeData(dataReader: GpuOrcDataReader, out: HostMemoryOutputStream, inputDataRanges: DiskRangeList): Unit
Copy the stripe to the channel
Copy the stripe to the channel
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
var
currentFileHostBuffers: Option[HostMemoryBuffersWithMetaDataBase]
- Attributes
- protected
- Definition Classes
- MultiFileCloudPartitionReaderBase
-
def
currentMetricsValues(): Array[CustomTaskMetric]
- Definition Classes
- PartitionReader
-
val
debugDumpAlways: Boolean
Whether to always debug dump or only on errors
Whether to always debug dump or only on errors
- Definition Classes
- MultiFileCloudOrcPartitionReader → OrcCommonFunctions
-
val
debugDumpPrefix: Option[String]
Whether debug dumping is enabled and the path prefix where to dump
Whether debug dumping is enabled and the path prefix where to dump
- Definition Classes
- MultiFileCloudOrcPartitionReader → OrcCommonFunctions
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
estimateOutputSizeFromBlocks(blocks: Seq[OrcStripeWithMeta]): Long
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
def
fileSystemBytesRead(): Long
- Attributes
- protected
- Definition Classes
- MultiFileReaderFunctions
- Annotations
- @nowarn()
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
def
get(): ColumnarBatch
- Definition Classes
- FilePartitionReaderBase → PartitionReader
-
def
getBatchRunner(tc: TaskContext, file: PartitionedFile, origFile: Option[PartitionedFile], conf: Configuration, filters: Array[Filter]): Callable[HostMemoryBuffersWithMetaDataBase]
The sub-class must implement the real file reading logic in a Callable which will be running in a thread pool
The sub-class must implement the real file reading logic in a Callable which will be running in a thread pool
- tc
task context to use
- file
file to be read
- origFile
optional original unmodified file if replaced with Alluxio
- conf
the Configuration parameters
- filters
push down filters
- returns
Callable[HostMemoryBuffersWithMetaDataBase]
- Definition Classes
- MultiFileCloudOrcPartitionReader → MultiFileCloudPartitionReaderBase
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getFileFormatShortName: String
File format short name used for logging and other things to uniquely identity which file format is being used.
File format short name used for logging and other things to uniquely identity which file format is being used.
- returns
the file format short name
- Definition Classes
- MultiFileCloudOrcPartitionReader → MultiFileCloudPartitionReaderBase
-
def
getORCOptionsAndSchema(memFileSchema: TypeDescription, requestedMapping: Option[Array[Int]], readDataSchema: StructType): (ORCOptions, TypeDescription)
- Definition Classes
- OrcCommonFunctions
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
var
isDone: Boolean
- Attributes
- protected
- Definition Classes
- FilePartitionReaderBase
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
final
def
isNeedToSplitDataBlock(curMeta: OrcBlockMetaForSplitCheck, nextMeta: OrcBlockMetaForSplitCheck): Boolean
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
val
metrics: Map[String, GpuMetric]
- Definition Classes
- ScanWithMetrics
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
next(): Boolean
- Definition Classes
- MultiFileCloudPartitionReaderBase → PartitionReader
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
populateCurrentBlockChunk(blockIterator: BufferedIterator[OrcOutputStripe], maxReadBatchSizeRows: Int, maxReadBatchSizeBytes: Long): Seq[OrcOutputStripe]
- Definition Classes
- OrcPartitionReaderBase
-
def
readBatches(fileBufsAndMeta: HostMemoryBuffersWithMetaDataBase): Iterator[ColumnarBatch]
Decode HostMemoryBuffers in GPU
Decode HostMemoryBuffers in GPU
- fileBufsAndMeta
the file HostMemoryBuffer read from a PartitionedFile
- returns
the decoded batches
- Definition Classes
- MultiFileCloudOrcPartitionReader → MultiFileCloudPartitionReaderBase
-
val
readDataSchema: StructType
- Definition Classes
- MultiFileCloudOrcPartitionReader → OrcCommonFunctions
-
def
readPartFile(ctx: OrcPartitionReaderContext, stripes: Seq[OrcOutputStripe]): (HostMemoryBuffer, Long)
Read the stripes into HostMemoryBuffer.
Read the stripes into HostMemoryBuffer.
- ctx
the context to provide some necessary information
- stripes
a sequence of Stripe to be read into HostMemeoryBuffer
- returns
HostMemeoryBuffer and its data size
- Attributes
- protected
- Definition Classes
- OrcPartitionReaderBase
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
implicit
def
toDataStripes(stripes: Seq[DataBlockBase]): Seq[OrcStripeWithMeta]
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
implicit
def
toStripe(block: DataBlockBase): OrcStripeWithMeta
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
withCodecOutputStream[T](ctx: OrcPartitionReaderContext, out: OutputStream)(block: (OrcProtoWriterShim) ⇒ T): T
Executes the provided code block in the codec environment
Executes the provided code block in the codec environment
- Definition Classes
- OrcCodecWritingHelper
-
final
def
writeOrcFileHeader(outStream: HostMemoryOutputStream): Long
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions
-
final
def
writeOrcFileTail(outStream: HostMemoryOutputStream, ctx: OrcPartitionReaderContext, footerStartOffset: Long, stripes: Seq[OrcOutputStripe]): Unit
write the ORC file footer and PostScript
write the ORC file footer and PostScript
- Attributes
- protected
- Definition Classes
- OrcCommonFunctions