case class GpuOrcMultiFilePartitionReaderFactory(sqlConf: SQLConf, broadcastedConf: Broadcast[SerializableConfiguration], dataSchema: StructType, readDataSchema: StructType, partitionSchema: StructType, filters: Array[Filter], rapidsConf: RapidsConf, metrics: Map[String, GpuMetric], queryUsesInputFile: Boolean) extends MultiFilePartitionReaderFactoryBase with Product with Serializable
The multi-file partition reader factory for creating cloud reading or coalescing reading for ORC file format.
- sqlConf
the SQLConf
- broadcastedConf
the Hadoop configuration
- dataSchema
schema of the data
- readDataSchema
the Spark schema describing what will be read
- partitionSchema
schema of partitions.
- filters
filters on non-partition columns
- rapidsConf
the Rapids configuration
- metrics
the metrics
- queryUsesInputFile
this is a parameter to easily allow turning it off in GpuTransitionOverrides if InputFileName, InputFileBlockStart, or InputFileBlockLength are used
- Alphabetic
- By Inheritance
- GpuOrcMultiFilePartitionReaderFactory
- Serializable
- Product
- Equals
- MultiFilePartitionReaderFactoryBase
- Logging
- PartitionReaderFactory
- Serializable
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
GpuOrcMultiFilePartitionReaderFactory(sqlConf: SQLConf, broadcastedConf: Broadcast[SerializableConfiguration], dataSchema: StructType, readDataSchema: StructType, partitionSchema: StructType, filters: Array[Filter], rapidsConf: RapidsConf, metrics: Map[String, GpuMetric], queryUsesInputFile: Boolean)
- sqlConf
the SQLConf
- broadcastedConf
the Hadoop configuration
- dataSchema
schema of the data
- readDataSchema
the Spark schema describing what will be read
- partitionSchema
schema of partitions.
- filters
filters on non-partition columns
- rapidsConf
the Rapids configuration
- metrics
the metrics
- queryUsesInputFile
this is a parameter to easily allow turning it off in GpuTransitionOverrides if InputFileName, InputFileBlockStart, or InputFileBlockLength are used
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- val broadcastedConf: Broadcast[SerializableConfiguration]
-
def
buildBaseColumnarReaderForCloud(files: Array[PartitionedFile], conf: Configuration): PartitionReader[ColumnarBatch]
Build the PartitionReader for cloud reading
Build the PartitionReader for cloud reading
- files
files to be read
- conf
configuration
- returns
cloud reading PartitionReader
- Definition Classes
- GpuOrcMultiFilePartitionReaderFactory → MultiFilePartitionReaderFactoryBase
-
def
buildBaseColumnarReaderForCoalescing(files: Array[PartitionedFile], conf: Configuration): PartitionReader[ColumnarBatch]
Build the PartitionReader for coalescing reading
Build the PartitionReader for coalescing reading
- files
files to be read
- conf
the configuration
- returns
coalescing reading PartitionReader
- Definition Classes
- GpuOrcMultiFilePartitionReaderFactory → MultiFilePartitionReaderFactoryBase
-
val
canUseCoalesceFilesReader: Boolean
An abstract method to indicate if coalescing reading can be used
An abstract method to indicate if coalescing reading can be used
- Definition Classes
- GpuOrcMultiFilePartitionReaderFactory → MultiFilePartitionReaderFactoryBase
-
val
canUseMultiThreadReader: Boolean
An abstract method to indicate if cloud reading can be used
An abstract method to indicate if cloud reading can be used
- Definition Classes
- GpuOrcMultiFilePartitionReaderFactory → MultiFilePartitionReaderFactoryBase
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
createColumnarReader(partition: InputPartition): PartitionReader[ColumnarBatch]
- Definition Classes
- MultiFilePartitionReaderFactoryBase → PartitionReaderFactory
-
def
createReader(partition: InputPartition): PartitionReader[InternalRow]
- Definition Classes
- MultiFilePartitionReaderFactoryBase → PartitionReaderFactory
- val dataSchema: StructType
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- val filters: Array[Filter]
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
getFileFormatShortName: String
File format short name used for logging and other things to uniquely identity which file format is being used.
File format short name used for logging and other things to uniquely identity which file format is being used.
- returns
the file format short name
- Definition Classes
- GpuOrcMultiFilePartitionReaderFactory → MultiFilePartitionReaderFactoryBase
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
val
maxChunkedReaderMemoryUsageSizeBytes: Long
- Attributes
- protected
- Definition Classes
- MultiFilePartitionReaderFactoryBase
-
val
maxGpuColumnSizeBytes: Long
- Attributes
- protected
- Definition Classes
- MultiFilePartitionReaderFactoryBase
-
val
maxReadBatchSizeBytes: Long
- Attributes
- protected
- Definition Classes
- MultiFilePartitionReaderFactoryBase
-
val
maxReadBatchSizeRows: Int
- Attributes
- protected
- Definition Classes
- MultiFilePartitionReaderFactoryBase
- val metrics: Map[String, GpuMetric]
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- val partitionSchema: StructType
- val queryUsesInputFile: Boolean
- val rapidsConf: RapidsConf
- val readDataSchema: StructType
- val sqlConf: SQLConf
-
def
supportColumnarReads(partition: InputPartition): Boolean
- Definition Classes
- MultiFilePartitionReaderFactoryBase → PartitionReaderFactory
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
val
targetBatchSizeBytes: Long
- Attributes
- protected
- Definition Classes
- MultiFilePartitionReaderFactoryBase
-
val
useChunkedReader: Boolean
- Attributes
- protected
- Definition Classes
- MultiFilePartitionReaderFactoryBase
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()