org.apache.parquet.hadoop.metadata

Class ColumnChunkMetaData

java.lang.Object

org.apache.parquet.hadoop.metadata.ColumnChunkMetaData

public abstract class ColumnChunkMetaData
extends Object

Column meta data for a block stored in the file footer and passed in the InputSplit

Field Summary

Fields

Modifier and Type	Field and Description
protected int	rowGroupOrdinal

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	ColumnChunkMetaData(ColumnChunkProperties columnChunkProperties)
protected	ColumnChunkMetaData(EncodingStats encodingStats, ColumnChunkProperties columnChunkProperties)

Method Summary

Modifier and Type	Method and Description
protected void	decryptIfNeeded()
static ColumnChunkMetaData	get(ColumnPath path, PrimitiveType.PrimitiveTypeName type, CompressionCodecName codec, EncodingStats encodingStats, Set<Encoding> encodings, Statistics statistics, long firstDataPage, long dictionaryPageOffset, long valueCount, long totalSize, long totalUncompressedSize) Deprecated. will be removed in 2.0.0. Use get(ColumnPath, PrimitiveType, CompressionCodecName, EncodingStats, Set, Statistics, long, long, long, long, long) instead.
static ColumnChunkMetaData	get(ColumnPath path, PrimitiveType.PrimitiveTypeName type, CompressionCodecName codec, Set<Encoding> encodings, long firstDataPage, long dictionaryPageOffset, long valueCount, long totalSize, long totalUncompressedSize) Deprecated.
static ColumnChunkMetaData	get(ColumnPath path, PrimitiveType.PrimitiveTypeName type, CompressionCodecName codec, Set<Encoding> encodings, Statistics statistics, long firstDataPage, long dictionaryPageOffset, long valueCount, long totalSize, long totalUncompressedSize) Deprecated.
static ColumnChunkMetaData	get(ColumnPath path, PrimitiveType type, CompressionCodecName codec, EncodingStats encodingStats, Set<Encoding> encodings, Statistics statistics, long firstDataPage, long dictionaryPageOffset, long valueCount, long totalSize, long totalUncompressedSize)
static ColumnChunkMetaData	get(ColumnPath path, PrimitiveType type, CompressionCodecName codec, EncodingStats encodingStats, Set<Encoding> encodings, Statistics statistics, long firstDataPage, long dictionaryPageOffset, long valueCount, long totalSize, long totalUncompressedSize, SizeStatistics sizeStatistics)
int	getBloomFilterLength()
long	getBloomFilterOffset()
CompressionCodecName	getCodec()
IndexReference	getColumnIndexReference() Method should be considered private
abstract long	getDictionaryPageOffset()
Set<Encoding>	getEncodings()
EncodingStats	getEncodingStats()
abstract long	getFirstDataPageOffset()
IndexReference	getOffsetIndexReference() Method should be considered private
ColumnPath	getPath()
PrimitiveType	getPrimitiveType()
int	getRowGroupOrdinal()
SizeStatistics	getSizeStatistics() Method should be considered private
long	getStartingPos()
abstract Statistics	getStatistics()
abstract long	getTotalSize()
abstract long	getTotalUncompressedSize()
PrimitiveType.PrimitiveTypeName	getType() Deprecated. will be removed in 2.0.0. Use getPrimitiveType() instead.
abstract long	getValueCount()
static ColumnChunkMetaData	getWithEncryptedMetadata(ParquetMetadataConverter parquetMetadataConverter, ColumnPath path, PrimitiveType type, byte[] encryptedMetadata, byte[] columnKeyMetadata, InternalFileDecryptor fileDecryptor, int rowGroupOrdinal, int columnOrdinal, String createdBy)
boolean	hasDictionaryPage()
boolean	isEncrypted() Method should be considered private
protected static boolean	positiveLongFitsInAnInt(long value) checks that a positive long value fits in an int.
void	setBloomFilterLength(int bloomFilterLength)
void	setBloomFilterOffset(long bloomFilterOffset) Method should be considered private
void	setColumnIndexReference(IndexReference indexReference) Method should be considered private
void	setOffsetIndexReference(IndexReference offsetIndexReference) Method should be considered private
void	setRowGroupOrdinal(int rowGroupOrdinal)
String	toString()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

rowGroupOrdinal

protected int rowGroupOrdinal

Constructor Detail

ColumnChunkMetaData

protected ColumnChunkMetaData(ColumnChunkProperties columnChunkProperties)

ColumnChunkMetaData

protected ColumnChunkMetaData(EncodingStats encodingStats,
                              ColumnChunkProperties columnChunkProperties)

Method Detail

get

@Deprecated
public static ColumnChunkMetaData get(ColumnPath path,
                                                  PrimitiveType.PrimitiveTypeName type,
                                                  CompressionCodecName codec,
                                                  Set<Encoding> encodings,
                                                  long firstDataPage,
                                                  long dictionaryPageOffset,
                                                  long valueCount,
                                                  long totalSize,
                                                  long totalUncompressedSize)

Deprecated.

get

@Deprecated
public static ColumnChunkMetaData get(ColumnPath path,
                                                  PrimitiveType.PrimitiveTypeName type,
                                                  CompressionCodecName codec,
                                                  Set<Encoding> encodings,
                                                  Statistics statistics,
                                                  long firstDataPage,
                                                  long dictionaryPageOffset,
                                                  long valueCount,
                                                  long totalSize,
                                                  long totalUncompressedSize)

Deprecated.

get

@Deprecated
public static ColumnChunkMetaData get(ColumnPath path,
                                                  PrimitiveType.PrimitiveTypeName type,
                                                  CompressionCodecName codec,
                                                  EncodingStats encodingStats,
                                                  Set<Encoding> encodings,
                                                  Statistics statistics,
                                                  long firstDataPage,
                                                  long dictionaryPageOffset,
                                                  long valueCount,
                                                  long totalSize,
                                                  long totalUncompressedSize)

Deprecated. will be removed in 2.0.0. Use get(ColumnPath, PrimitiveType, CompressionCodecName, EncodingStats, Set, Statistics, long, long, long, long, long) instead.

Parameters:

path - the path of this column in the write schema

type - primitive type for this column

codec - the compression codec used to compress

encodingStats - EncodingStats for the encodings used in this column

encodings - a set of encoding used in this column

statistics - statistics for the data in this column

firstDataPage - offset of the first non-dictionary page

dictionaryPageOffset - offset of the the dictionary page

valueCount - number of values

totalSize - total compressed size

totalUncompressedSize - uncompressed data size

Returns:

a column chunk metadata instance

get

public static ColumnChunkMetaData get(ColumnPath path,
                                      PrimitiveType type,
                                      CompressionCodecName codec,
                                      EncodingStats encodingStats,
                                      Set<Encoding> encodings,
                                      Statistics statistics,
                                      long firstDataPage,
                                      long dictionaryPageOffset,
                                      long valueCount,
                                      long totalSize,
                                      long totalUncompressedSize)

get

public static ColumnChunkMetaData get(ColumnPath path,
                                      PrimitiveType type,
                                      CompressionCodecName codec,
                                      EncodingStats encodingStats,
                                      Set<Encoding> encodings,
                                      Statistics statistics,
                                      long firstDataPage,
                                      long dictionaryPageOffset,
                                      long valueCount,
                                      long totalSize,
                                      long totalUncompressedSize,
                                      SizeStatistics sizeStatistics)

Parameters:

path - the path of this column in the write schema

type - primitive type for this column

codec - the compression codec used to compress

encodingStats - EncodingStats for the encodings used in this column

encodings - a set of encoding used in this column

statistics - statistics for the data in this column

firstDataPage - offset of the first non-dictionary page

dictionaryPageOffset - offset of the dictionary page

valueCount - number of values

totalSize - total compressed size

totalUncompressedSize - uncompressed data size

sizeStatistics - size statistics for the data in this column

Returns:

a column chunk metadata instance

getWithEncryptedMetadata

public static ColumnChunkMetaData getWithEncryptedMetadata(ParquetMetadataConverter parquetMetadataConverter,
                                                           ColumnPath path,
                                                           PrimitiveType type,
                                                           byte[] encryptedMetadata,
                                                           byte[] columnKeyMetadata,
                                                           InternalFileDecryptor fileDecryptor,
                                                           int rowGroupOrdinal,
                                                           int columnOrdinal,
                                                           String createdBy)

setRowGroupOrdinal

public void setRowGroupOrdinal(int rowGroupOrdinal)

getRowGroupOrdinal

public int getRowGroupOrdinal()

getStartingPos

public long getStartingPos()

Returns:

the offset of the first byte in the chunk

positiveLongFitsInAnInt

protected static boolean positiveLongFitsInAnInt(long value)

checks that a positive long value fits in an int. (reindexed on Integer.MIN_VALUE)

Parameters:

value - a long value

Returns:

whether it fits

decryptIfNeeded

protected void decryptIfNeeded()

getCodec

public CompressionCodecName getCodec()

getPath

public ColumnPath getPath()

Returns:

column identifier

getType

@Deprecated
public PrimitiveType.PrimitiveTypeName getType()

Deprecated. will be removed in 2.0.0. Use getPrimitiveType() instead.

Returns:

type of the column

getPrimitiveType

public PrimitiveType getPrimitiveType()

Returns:

the primitive type object of the column

getFirstDataPageOffset

public abstract long getFirstDataPageOffset()

Returns:

start of the column data offset

getDictionaryPageOffset

public abstract long getDictionaryPageOffset()

Returns:

the location of the dictionary page if any; 0 is returned if there is no dictionary page. Check hasDictionaryPage() to validate.

getValueCount

public abstract long getValueCount()

Returns:

count of values in this block of the column

getTotalUncompressedSize

public abstract long getTotalUncompressedSize()

Returns:

the totalUncompressedSize

getTotalSize

public abstract long getTotalSize()

Returns:

the totalSize

getStatistics

public abstract Statistics getStatistics()

Returns:

the stats for this column

getSizeStatistics

public SizeStatistics getSizeStatistics()

Method should be considered private

Returns:

the size stats for this column

getColumnIndexReference

public IndexReference getColumnIndexReference()

Method should be considered private

Returns:

the reference to the column index

setColumnIndexReference

public void setColumnIndexReference(IndexReference indexReference)

Method should be considered private

Parameters:

indexReference - the reference to the column index

getOffsetIndexReference

public IndexReference getOffsetIndexReference()

Method should be considered private

Returns:

the reference to the offset index

setOffsetIndexReference

public void setOffsetIndexReference(IndexReference offsetIndexReference)

Method should be considered private

Parameters:

offsetIndexReference - the reference to the offset index

setBloomFilterOffset

public void setBloomFilterOffset(long bloomFilterOffset)

Method should be considered private

Parameters:

bloomFilterOffset - the reference to the Bloom filter

setBloomFilterLength

public void setBloomFilterLength(int bloomFilterLength)

Parameters:

bloomFilterLength - the reference to the Bloom filter

getBloomFilterOffset

public long getBloomFilterOffset()

Returns:

the offset to the Bloom filter or -1 if there is no bloom filter for this column chunk

getBloomFilterLength

public int getBloomFilterLength()

Returns:

the length to the Bloom filter or -1 if there is no bloom filter length for this column chunk

getEncodings

public Set<Encoding> getEncodings()

Returns:

all the encodings used in this column

getEncodingStats

public EncodingStats getEncodingStats()

toString

public String toString()

Overrides:

toString in class Object

hasDictionaryPage

public boolean hasDictionaryPage()

isEncrypted

public boolean isEncrypted()

Method should be considered private

Returns:

whether or not this column is encrypted