c

io.delta.hive

DeltaRecordReaderWrapper

class DeltaRecordReaderWrapper extends ParquetRecordReaderWrapper

A record reader that reads data from the underlying Parquet reader and inserts partition values which don't exist in the Parquet files.

As we have verified the Hive schema in metastore is consistent with the Delta schema, the row returned by the underlying Parquet reader will match the Delta schema except that it leaves all partition columns as null since they are not in the raw parquet files. Hence, for the missing partition values, we need to use the partition information in DeltaInputSplit to create the corresponding Writables, and insert them into the corresponding positions when reading a row.

Linear Supertypes
ParquetRecordReaderWrapper, StatsProvidingRecordReader, RecordReader[NullWritable, ArrayWritable], Closeable, AutoCloseable, ParquetRecordReaderBase, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. DeltaRecordReaderWrapper
  2. ParquetRecordReaderWrapper
  3. StatsProvidingRecordReader
  4. RecordReader
  5. Closeable
  6. AutoCloseable
  7. ParquetRecordReaderBase
  8. AnyRef
  9. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new DeltaRecordReaderWrapper(inputFormat: ParquetInputFormat[ArrayWritable], split: DeltaInputSplit, jobConf: JobConf, reporter: Reporter)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  6. def close(): Unit
    Definition Classes
    ParquetRecordReaderWrapper → RecordReader → Closeable → AutoCloseable
    Annotations
    @throws( classOf[java.io.IOException] )
  7. def createKey(): NullWritable
    Definition Classes
    ParquetRecordReaderWrapper → RecordReader
  8. def createValue(): ArrayWritable
    Definition Classes
    ParquetRecordReaderWrapper → RecordReader
  9. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  10. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  11. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  12. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  13. def getFiltedBlocks(): List[BlockMetaData]
    Definition Classes
    ParquetRecordReaderBase
  14. def getPos(): Long
    Definition Classes
    ParquetRecordReaderWrapper → RecordReader
    Annotations
    @throws( classOf[java.io.IOException] )
  15. def getProgress(): Float
    Definition Classes
    ParquetRecordReaderWrapper → RecordReader
    Annotations
    @throws( classOf[java.io.IOException] )
  16. def getSplit(arg0: InputSplit, arg1: JobConf): ParquetInputSplit
    Attributes
    protected[parquet]
    Definition Classes
    ParquetRecordReaderBase
    Annotations
    @throws( classOf[java.io.IOException] )
  17. def getStats(): SerDeStats
    Definition Classes
    ParquetRecordReaderBase
  18. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  19. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  20. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  21. def next(key: NullWritable, value: ArrayWritable): Boolean
    Definition Classes
    DeltaRecordReaderWrapper → ParquetRecordReaderWrapper → RecordReader
  22. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  23. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  24. def setFilter(arg0: JobConf, arg1: MessageType): Filter
    Definition Classes
    ParquetRecordReaderBase
  25. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  26. def toString(): String
    Definition Classes
    AnyRef → Any
  27. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  28. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  29. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from ParquetRecordReaderWrapper

Inherited from StatsProvidingRecordReader

Inherited from RecordReader[NullWritable, ArrayWritable]

Inherited from Closeable

Inherited from AutoCloseable

Inherited from ParquetRecordReaderBase

Inherited from AnyRef

Inherited from Any

Ungrouped