org.apache.hadoop.hive.ql.io.parquet

Class ParquetRecordReaderBase

java.lang.Object

org.apache.hadoop.hive.ql.io.parquet.ParquetRecordReaderBase

Direct Known Subclasses:

ParquetRecordReaderWrapper, VectorizedParquetRecordReader

public abstract class ParquetRecordReaderBase
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.fs.Path	filePath
protected org.apache.hadoop.mapred.FileSplit	fileSplit
protected List<org.apache.parquet.hadoop.metadata.BlockMetaData>	filteredBlocks
protected org.apache.hadoop.mapred.JobConf	jobConf
protected Boolean	legacyConversionEnabled
static org.slf4j.Logger	LOG
protected org.apache.parquet.hadoop.ParquetInputSplit	parquetInputSplit
protected org.apache.parquet.hadoop.metadata.ParquetMetadata	parquetMetadata
protected ProjectionPusher	projectionPusher
protected org.apache.parquet.hadoop.ParquetFileReader	reader
protected int	schemaSize
protected SerDeStats	serDeStats
protected Boolean	skipProlepticConversion
protected boolean	skipTimestampConversion

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	ParquetRecordReaderBase(org.apache.hadoop.mapred.JobConf conf, org.apache.hadoop.mapred.InputSplit oldSplit)

Method Summary

Modifier and Type	Method and Description
List<org.apache.parquet.hadoop.metadata.BlockMetaData>	getFilteredBlocks()
protected org.apache.parquet.hadoop.metadata.ParquetMetadata	getParquetMetadata(org.apache.hadoop.fs.Path path, org.apache.hadoop.mapred.JobConf conf)
protected org.apache.parquet.hadoop.ParquetInputSplit	getSplit(org.apache.hadoop.mapred.JobConf conf) gets a ParquetInputSplit corresponding to a split given by Hive
SerDeStats	getStats()
org.apache.parquet.filter2.compat.FilterCompat.Filter	setFilter(org.apache.hadoop.mapred.JobConf conf, org.apache.parquet.schema.MessageType schema)
protected void	setupMetadataAndParquetSplit(org.apache.hadoop.mapred.JobConf conf, org.apache.parquet.hadoop.metadata.ParquetMetadata metadata)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

public static final org.slf4j.Logger LOG

fileSplit

protected final org.apache.hadoop.mapred.FileSplit fileSplit

filePath

protected org.apache.hadoop.fs.Path filePath

parquetInputSplit

protected org.apache.parquet.hadoop.ParquetInputSplit parquetInputSplit

parquetMetadata

protected org.apache.parquet.hadoop.metadata.ParquetMetadata parquetMetadata

projectionPusher

protected ProjectionPusher projectionPusher

skipTimestampConversion

protected boolean skipTimestampConversion

skipProlepticConversion

protected Boolean skipProlepticConversion

legacyConversionEnabled

protected Boolean legacyConversionEnabled

serDeStats

protected SerDeStats serDeStats

jobConf

protected final org.apache.hadoop.mapred.JobConf jobConf

schemaSize

protected int schemaSize

filteredBlocks

protected List<org.apache.parquet.hadoop.metadata.BlockMetaData> filteredBlocks

reader

protected org.apache.parquet.hadoop.ParquetFileReader reader

Constructor Detail

ParquetRecordReaderBase

protected ParquetRecordReaderBase(org.apache.hadoop.mapred.JobConf conf,
                                  org.apache.hadoop.mapred.InputSplit oldSplit)
                           throws IOException

Throws:

IOException

Method Detail

setupMetadataAndParquetSplit

protected void setupMetadataAndParquetSplit(org.apache.hadoop.mapred.JobConf conf,
                                            org.apache.parquet.hadoop.metadata.ParquetMetadata metadata)
                                     throws IOException

Throws:

IOException

getSplit

protected org.apache.parquet.hadoop.ParquetInputSplit getSplit(org.apache.hadoop.mapred.JobConf conf)
                                                        throws IOException

gets a ParquetInputSplit corresponding to a split given by Hive

Parameters:

conf - The JobConf of the Hive job

Returns:

a ParquetInputSplit corresponding to the oldSplit

Throws:

IOException - if the config cannot be enhanced or if the footer cannot be read from the file

getParquetMetadata

protected org.apache.parquet.hadoop.metadata.ParquetMetadata getParquetMetadata(org.apache.hadoop.fs.Path path,
                                                                                org.apache.hadoop.mapred.JobConf conf)
                                                                         throws IOException

Throws:

IOException

setFilter

public org.apache.parquet.filter2.compat.FilterCompat.Filter setFilter(org.apache.hadoop.mapred.JobConf conf,
                                                                       org.apache.parquet.schema.MessageType schema)

getFilteredBlocks

public List<org.apache.parquet.hadoop.metadata.BlockMetaData> getFilteredBlocks()

getStats

public SerDeStats getStats()