Packages

package parquet

Type Members

  1. class ParquetDataSourceV2 extends FileDataSourceV2
  2. case class ParquetPartitionReaderFactory(sqlConf: SQLConf, broadcastedConf: Broadcast[SerializableConfiguration], dataSchema: StructType, readDataSchema: StructType, partitionSchema: StructType, filters: Array[Filter], aggregation: Option[Aggregation], parquetOptions: ParquetOptions) extends FilePartitionReaderFactory with Logging with Product with Serializable

    A factory used to create Parquet readers.

    A factory used to create Parquet readers.

    sqlConf

    SQL configuration.

    broadcastedConf

    Broadcast serializable Hadoop Configuration.

    dataSchema

    Schema of Parquet files.

    readDataSchema

    Required schema of Parquet files.

    partitionSchema

    Schema of partitions.

    filters

    Filters to be pushed down in the batch scan.

    aggregation

    Aggregation to be pushed down in the batch scan.

    parquetOptions

    The options of Parquet datasource that are set for the read.

  3. case class ParquetScan(sparkSession: SparkSession, hadoopConf: Configuration, fileIndex: PartitioningAwareFileIndex, dataSchema: StructType, readDataSchema: StructType, readPartitionSchema: StructType, pushedFilters: Array[Filter], options: CaseInsensitiveStringMap, pushedAggregate: Option[Aggregation] = None, partitionFilters: Seq[Expression] = Seq.empty, dataFilters: Seq[Expression] = Seq.empty) extends FileScan with Product with Serializable
  4. case class ParquetScanBuilder(sparkSession: SparkSession, fileIndex: PartitioningAwareFileIndex, schema: StructType, dataSchema: StructType, options: CaseInsensitiveStringMap) extends FileScanBuilder with SupportsPushDownAggregates with Product with Serializable
  5. case class ParquetTable(name: String, sparkSession: SparkSession, options: CaseInsensitiveStringMap, paths: Seq[String], userSpecifiedSchema: Option[StructType], fallbackFileFormat: Class[_ <: FileFormat]) extends FileTable with Product with Serializable
  6. case class ParquetWrite(paths: Seq[String], formatName: String, supportsDataType: (DataType) => Boolean, info: LogicalWriteInfo) extends FileWrite with Logging with Product with Serializable

Ungrouped