org.apache.spark.sql.execution.datasources.parquet

Class Spark24HoodieVectorizedParquetRecordReader

java.lang.Object

org.apache.hadoop.mapreduce.RecordReader<Void,T>

org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase<Object>

org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

org.apache.spark.sql.execution.datasources.parquet.Spark24HoodieVectorizedParquetRecordReader

All Implemented Interfaces:

Closeable, AutoCloseable

public class Spark24HoodieVectorizedParquetRecordReader
extends org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase

org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase.NullIntIterator, org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase.RLEIntIterator, org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase.ValuesReaderIntIterator

Field Summary

Fields inherited from class org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase

file, fileSchema, reader, requestedSchema, sparkSchema, totalRowCount

Constructor Summary

Constructors

Constructor and Description
Spark24HoodieVectorizedParquetRecordReader(TimeZone convertTz, boolean useOffHeap, int capacity, Map<Integer,org.apache.hudi.common.util.collection.Pair<org.apache.spark.sql.types.DataType,org.apache.spark.sql.types.DataType>> typeChangeInfos)

Method Summary

Modifier and Type	Method and Description
void	close()
void	enableReturningBatches()
Object	getCurrentValue()
void	initBatch(org.apache.spark.sql.types.StructType partitionColumns, org.apache.spark.sql.catalyst.InternalRow partitionValues)
void	initialize(org.apache.hadoop.mapreduce.InputSplit inputSplit, org.apache.hadoop.mapreduce.TaskAttemptContext taskAttemptContext)
boolean	nextBatch()
boolean	nextKeyValue()
org.apache.spark.sql.vectorized.ColumnarBatch	resultBatch()

Methods inherited from class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

getProgress, initialize

Methods inherited from class org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase

createRLEIterator, getCurrentKey, listDirectory

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

Spark24HoodieVectorizedParquetRecordReader

public Spark24HoodieVectorizedParquetRecordReader(TimeZone convertTz,
                                                  boolean useOffHeap,
                                                  int capacity,
                                                  Map<Integer,org.apache.hudi.common.util.collection.Pair<org.apache.spark.sql.types.DataType,org.apache.spark.sql.types.DataType>> typeChangeInfos)

Method Detail

initBatch

public void initBatch(org.apache.spark.sql.types.StructType partitionColumns,
                      org.apache.spark.sql.catalyst.InternalRow partitionValues)

Overrides:

initBatch in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

initialize

public void initialize(org.apache.hadoop.mapreduce.InputSplit inputSplit,
                       org.apache.hadoop.mapreduce.TaskAttemptContext taskAttemptContext)
                throws IOException,
                       InterruptedException,
                       UnsupportedOperationException

Overrides:

initialize in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

Throws:

IOException

InterruptedException

UnsupportedOperationException

close

public void close()
           throws IOException

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

Overrides:

close in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

Throws:

IOException

resultBatch

public org.apache.spark.sql.vectorized.ColumnarBatch resultBatch()

Overrides:

resultBatch in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

nextBatch

public boolean nextBatch()
                  throws IOException

Overrides:

nextBatch in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

Throws:

IOException

enableReturningBatches

public void enableReturningBatches()

Overrides:

enableReturningBatches in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

getCurrentValue

public Object getCurrentValue()

Overrides:

getCurrentValue in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

nextKeyValue

public boolean nextKeyValue()
                     throws IOException

Overrides:

nextKeyValue in class org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader

Throws:

IOException