org.apache.parquet.format.converter

Class ParquetMetadataConverter

java.lang.Object

org.apache.parquet.format.converter.ParquetMetadataConverter

public class ParquetMetadataConverter
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	ParquetMetadataConverter.MetadataFilter

Field Summary

Fields

Modifier and Type	Field and Description
static long	MAX_STATS_SIZE
static ParquetMetadataConverter.MetadataFilter	NO_FILTER
static ParquetMetadataConverter.MetadataFilter	SKIP_ROW_GROUPS

Constructor Summary

Constructors

Constructor and Description
ParquetMetadataConverter()
ParquetMetadataConverter(org.apache.hadoop.conf.Configuration conf) Deprecated. will be removed in 2.0.0; use ParquetMetadataConverter(ParquetReadOptions)
ParquetMetadataConverter(int statisticsTruncateLength)
ParquetMetadataConverter(ParquetReadOptions options)

Method Summary

Modifier and Type	Method and Description
ColumnChunkMetaData	buildColumnChunkMetaData(ColumnMetaData metaData, ColumnPath columnPath, PrimitiveType type, String createdBy)
List<PageEncodingStats>	convertEncodingStats(EncodingStats stats)
EncodingStats	convertEncodingStats(List<PageEncodingStats> stats)
static ColumnIndex	fromParquetColumnIndex(PrimitiveType type, ColumnIndex parquetColumnIndex)
ParquetMetadata	fromParquetMetadata(FileMetaData parquetMetadata)
ParquetMetadata	fromParquetMetadata(FileMetaData parquetMetadata, InternalFileDecryptor fileDecryptor, boolean encryptedFooter)
ParquetMetadata	fromParquetMetadata(FileMetaData parquetMetadata, InternalFileDecryptor fileDecryptor, boolean encryptedFooter, Map<RowGroup,Long> rowGroupToRowIndexOffsetMap)
static OffsetIndex	fromParquetOffsetIndex(OffsetIndex parquetOffsetIndex)
static SizeStatistics	fromParquetSizeStatistics(SizeStatistics statistics, PrimitiveType type)
static Statistics	fromParquetStatistics(Statistics statistics, PrimitiveType.PrimitiveTypeName type) Deprecated. will be removed in 2.0.0.
static Statistics	fromParquetStatistics(String createdBy, Statistics statistics, PrimitiveType.PrimitiveTypeName type) Deprecated. will be removed in 2.0.0.
Statistics	fromParquetStatistics(String createdBy, Statistics statistics, PrimitiveType type)
Encoding	getEncoding(Encoding encoding)
Encoding	getEncoding(Encoding encoding)
PrimitiveType.PrimitiveTypeName	getPrimitive(Type type)
static ParquetMetadataConverter.MetadataFilter	offsets(long... offsets)
static ParquetMetadataConverter.MetadataFilter	range(long startOffset, long endOffset) [ startOffset, endOffset )
ParquetMetadata	readParquetMetadata(InputStream from) Deprecated.
ParquetMetadata	readParquetMetadata(InputStream from, ParquetMetadataConverter.MetadataFilter filter)
ParquetMetadata	readParquetMetadata(InputStream fromInputStream, ParquetMetadataConverter.MetadataFilter filter, InternalFileDecryptor fileDecryptor, boolean encryptedFooter, int combinedFooterLength)
static BloomFilterHeader	toBloomFilterHeader(BloomFilter bloomFilter)
static ColumnIndex	toParquetColumnIndex(PrimitiveType type, ColumnIndex columnIndex)
FileMetaData	toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata)
FileMetaData	toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata, InternalFileEncryptor fileEncryptor)
static OffsetIndex	toParquetOffsetIndex(OffsetIndex offsetIndex)
static SizeStatistics	toParquetSizeStatistics(SizeStatistics stats)
static Statistics	toParquetStatistics(Statistics stats)
static Statistics	toParquetStatistics(Statistics stats, int truncateLength)
void	writeDataPageHeader(int uncompressedSize, int compressedSize, int valueCount, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding, OutputStream to) Deprecated.
void	writeDataPageHeader(int uncompressedSize, int compressedSize, int valueCount, Statistics statistics, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding, OutputStream to) Deprecated.
void	writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding, int crc, OutputStream to)
void	writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding, int crc, OutputStream to, BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void	writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding, OutputStream to)
void	writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding, OutputStream to, BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void	writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, Encoding dataEncoding, int rlByteLength, int dlByteLength, int crc, OutputStream to, BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void	writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to)
void	writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to, BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void	writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, Statistics statistics, Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to) Deprecated.
void	writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, Encoding valuesEncoding, int crc, OutputStream to)
void	writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, Encoding valuesEncoding, int crc, OutputStream to, BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void	writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, Encoding valuesEncoding, OutputStream to)
void	writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, Encoding valuesEncoding, OutputStream to, BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

NO_FILTER

public static final ParquetMetadataConverter.MetadataFilter NO_FILTER

SKIP_ROW_GROUPS

public static final ParquetMetadataConverter.MetadataFilter SKIP_ROW_GROUPS

MAX_STATS_SIZE

public static final long MAX_STATS_SIZE

See Also:

Constant Field Values

Constructor Detail

ParquetMetadataConverter

public ParquetMetadataConverter()

ParquetMetadataConverter

public ParquetMetadataConverter(int statisticsTruncateLength)

ParquetMetadataConverter

@Deprecated
public ParquetMetadataConverter(org.apache.hadoop.conf.Configuration conf)

Deprecated. will be removed in 2.0.0; use ParquetMetadataConverter(ParquetReadOptions)

Parameters:

conf - a configuration

ParquetMetadataConverter

public ParquetMetadataConverter(ParquetReadOptions options)

Method Detail

toParquetMetadata

public FileMetaData toParquetMetadata(int currentVersion,
                                      ParquetMetadata parquetMetadata)

toParquetMetadata

public FileMetaData toParquetMetadata(int currentVersion,
                                      ParquetMetadata parquetMetadata,
                                      InternalFileEncryptor fileEncryptor)

getEncoding

public Encoding getEncoding(Encoding encoding)

getEncoding

public Encoding getEncoding(Encoding encoding)

convertEncodingStats

public EncodingStats convertEncodingStats(List<PageEncodingStats> stats)

convertEncodingStats

public List<PageEncodingStats> convertEncodingStats(EncodingStats stats)

toParquetStatistics

public static Statistics toParquetStatistics(Statistics stats)

toParquetStatistics

public static Statistics toParquetStatistics(Statistics stats,
                                             int truncateLength)

fromParquetStatistics

@Deprecated
public static Statistics fromParquetStatistics(Statistics statistics,
                                                           PrimitiveType.PrimitiveTypeName type)

Deprecated. will be removed in 2.0.0.

Parameters:

statistics - parquet format statistics

type - a primitive type name

Returns:

the statistics

fromParquetStatistics

@Deprecated
public static Statistics fromParquetStatistics(String createdBy,
                                                           Statistics statistics,
                                                           PrimitiveType.PrimitiveTypeName type)

Deprecated. will be removed in 2.0.0.

Parameters:

createdBy - the created-by string from the file

statistics - parquet format statistics

type - a primitive type name

Returns:

the statistics

fromParquetStatistics

public Statistics fromParquetStatistics(String createdBy,
                                        Statistics statistics,
                                        PrimitiveType type)

getPrimitive

public PrimitiveType.PrimitiveTypeName getPrimitive(Type type)

range

public static ParquetMetadataConverter.MetadataFilter range(long startOffset,
                                                            long endOffset)

[ startOffset, endOffset )

Parameters:

startOffset - a start offset (inclusive)

endOffset - an end offset (exclusive)

Returns:

a range filter from the offsets

offsets

public static ParquetMetadataConverter.MetadataFilter offsets(long... offsets)

readParquetMetadata

@Deprecated
public ParquetMetadata readParquetMetadata(InputStream from)
                                                throws IOException

Deprecated.

Throws:

IOException

readParquetMetadata

public ParquetMetadata readParquetMetadata(InputStream from,
                                           ParquetMetadataConverter.MetadataFilter filter)
                                    throws IOException

Throws:

IOException

readParquetMetadata

public ParquetMetadata readParquetMetadata(InputStream fromInputStream,
                                           ParquetMetadataConverter.MetadataFilter filter,
                                           InternalFileDecryptor fileDecryptor,
                                           boolean encryptedFooter,
                                           int combinedFooterLength)
                                    throws IOException

Throws:

IOException

buildColumnChunkMetaData

public ColumnChunkMetaData buildColumnChunkMetaData(ColumnMetaData metaData,
                                                    ColumnPath columnPath,
                                                    PrimitiveType type,
                                                    String createdBy)

fromParquetMetadata

public ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata)
                                    throws IOException

Throws:

IOException

fromParquetMetadata

public ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata,
                                           InternalFileDecryptor fileDecryptor,
                                           boolean encryptedFooter)
                                    throws IOException

Throws:

IOException

fromParquetMetadata

public ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata,
                                           InternalFileDecryptor fileDecryptor,
                                           boolean encryptedFooter,
                                           Map<RowGroup,Long> rowGroupToRowIndexOffsetMap)
                                    throws IOException

Throws:

IOException

writeDataPageHeader

@Deprecated
public void writeDataPageHeader(int uncompressedSize,
                                            int compressedSize,
                                            int valueCount,
                                            Encoding rlEncoding,
                                            Encoding dlEncoding,
                                            Encoding valuesEncoding,
                                            OutputStream to)
                                     throws IOException

Deprecated.

Throws:

IOException

writeDataPageHeader

@Deprecated
public void writeDataPageHeader(int uncompressedSize,
                                            int compressedSize,
                                            int valueCount,
                                            Statistics statistics,
                                            Encoding rlEncoding,
                                            Encoding dlEncoding,
                                            Encoding valuesEncoding,
                                            OutputStream to)
                                     throws IOException

Deprecated.

Throws:

IOException

writeDataPageV2Header

@Deprecated
public void writeDataPageV2Header(int uncompressedSize,
                                              int compressedSize,
                                              int valueCount,
                                              int nullCount,
                                              int rowCount,
                                              Statistics statistics,
                                              Encoding dataEncoding,
                                              int rlByteLength,
                                              int dlByteLength,
                                              OutputStream to)
                                       throws IOException

Deprecated.

Throws:

IOException

writeDataPageV1Header

public void writeDataPageV1Header(int uncompressedSize,
                                  int compressedSize,
                                  int valueCount,
                                  Encoding rlEncoding,
                                  Encoding dlEncoding,
                                  Encoding valuesEncoding,
                                  OutputStream to)
                           throws IOException

Throws:

IOException

writeDataPageV1Header

public void writeDataPageV1Header(int uncompressedSize,
                                  int compressedSize,
                                  int valueCount,
                                  Encoding rlEncoding,
                                  Encoding dlEncoding,
                                  Encoding valuesEncoding,
                                  OutputStream to,
                                  BlockCipher.Encryptor blockEncryptor,
                                  byte[] pageHeaderAAD)
                           throws IOException

Throws:

IOException

writeDataPageV1Header

public void writeDataPageV1Header(int uncompressedSize,
                                  int compressedSize,
                                  int valueCount,
                                  Encoding rlEncoding,
                                  Encoding dlEncoding,
                                  Encoding valuesEncoding,
                                  int crc,
                                  OutputStream to)
                           throws IOException

Throws:

IOException

writeDataPageV1Header

public void writeDataPageV1Header(int uncompressedSize,
                                  int compressedSize,
                                  int valueCount,
                                  Encoding rlEncoding,
                                  Encoding dlEncoding,
                                  Encoding valuesEncoding,
                                  int crc,
                                  OutputStream to,
                                  BlockCipher.Encryptor blockEncryptor,
                                  byte[] pageHeaderAAD)
                           throws IOException

Throws:

IOException

writeDataPageV2Header

public void writeDataPageV2Header(int uncompressedSize,
                                  int compressedSize,
                                  int valueCount,
                                  int nullCount,
                                  int rowCount,
                                  Encoding dataEncoding,
                                  int rlByteLength,
                                  int dlByteLength,
                                  OutputStream to)
                           throws IOException

Throws:

IOException

writeDataPageV2Header

public void writeDataPageV2Header(int uncompressedSize,
                                  int compressedSize,
                                  int valueCount,
                                  int nullCount,
                                  int rowCount,
                                  Encoding dataEncoding,
                                  int rlByteLength,
                                  int dlByteLength,
                                  OutputStream to,
                                  BlockCipher.Encryptor blockEncryptor,
                                  byte[] pageHeaderAAD)
                           throws IOException

Throws:

IOException

writeDataPageV2Header

public void writeDataPageV2Header(int uncompressedSize,
                                  int compressedSize,
                                  int valueCount,
                                  int nullCount,
                                  int rowCount,
                                  Encoding dataEncoding,
                                  int rlByteLength,
                                  int dlByteLength,
                                  int crc,
                                  OutputStream to,
                                  BlockCipher.Encryptor blockEncryptor,
                                  byte[] pageHeaderAAD)
                           throws IOException

Throws:

IOException

writeDictionaryPageHeader

public void writeDictionaryPageHeader(int uncompressedSize,
                                      int compressedSize,
                                      int valueCount,
                                      Encoding valuesEncoding,
                                      OutputStream to)
                               throws IOException

Throws:

IOException

writeDictionaryPageHeader

public void writeDictionaryPageHeader(int uncompressedSize,
                                      int compressedSize,
                                      int valueCount,
                                      Encoding valuesEncoding,
                                      OutputStream to,
                                      BlockCipher.Encryptor blockEncryptor,
                                      byte[] pageHeaderAAD)
                               throws IOException

Throws:

IOException

writeDictionaryPageHeader

public void writeDictionaryPageHeader(int uncompressedSize,
                                      int compressedSize,
                                      int valueCount,
                                      Encoding valuesEncoding,
                                      int crc,
                                      OutputStream to)
                               throws IOException

Throws:

IOException

writeDictionaryPageHeader

public void writeDictionaryPageHeader(int uncompressedSize,
                                      int compressedSize,
                                      int valueCount,
                                      Encoding valuesEncoding,
                                      int crc,
                                      OutputStream to,
                                      BlockCipher.Encryptor blockEncryptor,
                                      byte[] pageHeaderAAD)
                               throws IOException

Throws:

IOException

toParquetColumnIndex

public static ColumnIndex toParquetColumnIndex(PrimitiveType type,
                                               ColumnIndex columnIndex)

fromParquetColumnIndex

public static ColumnIndex fromParquetColumnIndex(PrimitiveType type,
                                                 ColumnIndex parquetColumnIndex)

toParquetOffsetIndex

public static OffsetIndex toParquetOffsetIndex(OffsetIndex offsetIndex)

fromParquetOffsetIndex

public static OffsetIndex fromParquetOffsetIndex(OffsetIndex parquetOffsetIndex)

toBloomFilterHeader

public static BloomFilterHeader toBloomFilterHeader(BloomFilter bloomFilter)

fromParquetSizeStatistics

public static SizeStatistics fromParquetSizeStatistics(SizeStatistics statistics,
                                                       PrimitiveType type)

toParquetSizeStatistics

public static SizeStatistics toParquetSizeStatistics(SizeStatistics stats)