A B C D E F G H I J K L M N O P Q R S T U V W Y Z

A

abortStagedChanges() - Method in class org.apache.iceberg.spark.RollbackStagedTable

abortStagedChanges() - Method in class org.apache.iceberg.spark.source.StagedSparkTable

accessor() - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

add(T) - Method in class org.apache.iceberg.spark.actions.SetAccumulator

add(String, Table) - Method in class org.apache.iceberg.spark.SparkTableCache

addedFilesCount() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

addedRowsCount() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

AGGREGATE_PUSH_DOWN_ENABLED - Static variable in class org.apache.iceberg.spark.SparkReadOptions

AGGREGATE_PUSH_DOWN_ENABLED - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

AGGREGATE_PUSH_DOWN_ENABLED_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

aggregatePushDownEnabled() - Method in class org.apache.iceberg.spark.SparkReadConf

aggregateTaskMetrics(long[]) - Method in class org.apache.iceberg.spark.source.metrics.NumDeletes

aggregateTaskMetrics(long[]) - Method in class org.apache.iceberg.spark.source.metrics.NumSplits

alterNamespace(String[], NamespaceChange...) - Method in class org.apache.iceberg.spark.SparkCatalog

alterNamespace(String[], NamespaceChange...) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

alterTable(Identifier, TableChange...) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

alterTable(Identifier, TableChange...) - Method in class org.apache.iceberg.spark.SparkCatalog

alterTable(Identifier, TableChange...) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

AncestorsOfProcedure - Class in org.apache.iceberg.spark.procedures

apply(int, int) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBase

applyPropertyChanges(UpdateProperties, List<TableChange>) - Static method in class org.apache.iceberg.spark.Spark3Util

Applies a list of Spark table changes to an UpdateProperties operation.

applySchemaChanges(UpdateSchema, List<TableChange>) - Static method in class org.apache.iceberg.spark.Spark3Util

Applies a list of Spark table changes to an UpdateSchema operation.

arrayElementType(DataType) - Method in class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

ArrowVectorAccessors - Class in org.apache.iceberg.spark.data.vectorized

as(String) - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

AS_OF_TIMESTAMP - Static variable in class org.apache.iceberg.spark.SparkReadOptions

asOfTimestamp() - Method in class org.apache.iceberg.spark.SparkReadConf

AvroWithSparkSchemaVisitor<T> - Class in org.apache.iceberg.spark.data

AvroWithSparkSchemaVisitor() - Constructor for class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

B

bind(StructType) - Method in class org.apache.iceberg.spark.functions.BucketFunction

bind(StructType) - Method in class org.apache.iceberg.spark.functions.IcebergVersionFunction

bind(StructType) - Method in class org.apache.iceberg.spark.functions.TruncateFunction

binPack() - Method in class org.apache.iceberg.spark.actions.RewriteDataFilesSparkAction

branch() - Method in class org.apache.iceberg.spark.SparkReadConf

BRANCH - Static variable in class org.apache.iceberg.spark.SparkReadOptions

branch() - Method in class org.apache.iceberg.spark.SparkWriteConf

BucketBase() - Constructor for class org.apache.iceberg.spark.functions.BucketFunction.BucketBase

BucketBinary() - Constructor for class org.apache.iceberg.spark.functions.BucketFunction.BucketBinary

BucketDecimal(DataType) - Constructor for class org.apache.iceberg.spark.functions.BucketFunction.BucketDecimal

BucketFunction - Class in org.apache.iceberg.spark.functions

A Spark function implementation for the Iceberg bucket transform.

BucketFunction() - Constructor for class org.apache.iceberg.spark.functions.BucketFunction

BucketFunction.BucketBase - Class in org.apache.iceberg.spark.functions

BucketFunction.BucketBinary - Class in org.apache.iceberg.spark.functions

BucketFunction.BucketDecimal - Class in org.apache.iceberg.spark.functions

BucketFunction.BucketInt - Class in org.apache.iceberg.spark.functions

BucketFunction.BucketLong - Class in org.apache.iceberg.spark.functions

BucketFunction.BucketString - Class in org.apache.iceberg.spark.functions

BucketInt(DataType) - Constructor for class org.apache.iceberg.spark.functions.BucketFunction.BucketInt

BucketLong(DataType) - Constructor for class org.apache.iceberg.spark.functions.BucketFunction.BucketLong

BucketString() - Constructor for class org.apache.iceberg.spark.functions.BucketFunction.BucketString

build() - Method in interface org.apache.iceberg.spark.procedures.SparkProcedures.ProcedureBuilder

build() - Method in class org.apache.iceberg.spark.source.SparkPositionDeletesRewriteBuilder

build() - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

build() - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaWriteBuilder

Returns a logical delta write.

buildChangelogScan() - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

buildCopyOnWriteDistribution(Table, RowLevelOperation.Command, DistributionMode) - Static method in class org.apache.iceberg.spark.SparkDistributionAndOrderingUtil

buildCopyOnWriteOrdering(Table, RowLevelOperation.Command, Distribution) - Static method in class org.apache.iceberg.spark.SparkDistributionAndOrderingUtil

buildCopyOnWriteScan() - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

builder() - Static method in class org.apache.iceberg.spark.procedures.AncestorsOfProcedure

builder() - Static method in class org.apache.iceberg.spark.procedures.CreateChangelogViewProcedure

builder() - Static method in class org.apache.iceberg.spark.procedures.ExpireSnapshotsProcedure

builder() - Static method in class org.apache.iceberg.spark.procedures.FastForwardBranchProcedure

builder() - Static method in class org.apache.iceberg.spark.procedures.RemoveOrphanFilesProcedure

builder() - Static method in class org.apache.iceberg.spark.procedures.RewritePositionDeleteFilesProcedure

buildIcebergCatalog(String, CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.SparkCatalog

Build an Iceberg Catalog to be used by this Spark catalog adapter.

buildIdentifier(Identifier) - Method in class org.apache.iceberg.spark.SparkCatalog

Build an Iceberg TableIdentifier for the given Spark identifier.

buildMergeOnReadScan() - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

buildPositionDeltaDistribution(Table, RowLevelOperation.Command, DistributionMode) - Static method in class org.apache.iceberg.spark.SparkDistributionAndOrderingUtil

buildPositionDeltaOrdering(Table, RowLevelOperation.Command) - Static method in class org.apache.iceberg.spark.SparkDistributionAndOrderingUtil

buildReader(Schema, MessageType) - Static method in class org.apache.iceberg.spark.data.SparkParquetReaders

buildReader(Schema, MessageType, Map<Integer, ?>) - Static method in class org.apache.iceberg.spark.data.SparkParquetReaders

buildReader(Schema, TypeDescription, Map<Integer, ?>) - Static method in class org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders

buildReader(Schema, MessageType, Map<Integer, ?>, DeleteFilter<InternalRow>) - Static method in class org.apache.iceberg.spark.data.vectorized.VectorizedSparkParquetReaders

buildRequiredDistribution(Table, DistributionMode) - Static method in class org.apache.iceberg.spark.SparkDistributionAndOrderingUtil

buildRequiredOrdering(Table, Distribution) - Static method in class org.apache.iceberg.spark.SparkDistributionAndOrderingUtil

buildSparkCatalog(String, CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

Build a SparkCatalog to be used for Iceberg operations.

buildWriter(StructType, MessageType) - Static method in class org.apache.iceberg.spark.data.SparkParquetWriters

C

call(InternalRow) - Method in class org.apache.iceberg.spark.procedures.AncestorsOfProcedure

call(InternalRow) - Method in class org.apache.iceberg.spark.procedures.CreateChangelogViewProcedure

call(InternalRow) - Method in class org.apache.iceberg.spark.procedures.ExpireSnapshotsProcedure

call(InternalRow) - Method in class org.apache.iceberg.spark.procedures.FastForwardBranchProcedure

call(InternalRow) - Method in class org.apache.iceberg.spark.procedures.RemoveOrphanFilesProcedure

call(InternalRow) - Method in class org.apache.iceberg.spark.procedures.RewritePositionDeleteFilesProcedure

call(InternalRow) - Method in interface org.apache.spark.sql.connector.iceberg.catalog.Procedure

Executes this procedure.

canDeleteWhere(Filter[]) - Method in class org.apache.iceberg.spark.source.SparkTable

canonicalName() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBinary

canonicalName() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketDecimal

canonicalName() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketInt

canonicalName() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketLong

canonicalName() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketString

canonicalName() - Method in class org.apache.iceberg.spark.functions.DaysFunction.DateToDaysFunction

canonicalName() - Method in class org.apache.iceberg.spark.functions.DaysFunction.TimestampToDaysFunction

canonicalName() - Method in class org.apache.iceberg.spark.functions.HoursFunction.TimestampToHoursFunction

canonicalName() - Method in class org.apache.iceberg.spark.functions.MonthsFunction.DateToMonthsFunction

canonicalName() - Method in class org.apache.iceberg.spark.functions.MonthsFunction.TimestampToMonthsFunction

canonicalName() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBigInt

canonicalName() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBinary

canonicalName() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateDecimal

canonicalName() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateInt

canonicalName() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateSmallInt

canonicalName() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateString

canonicalName() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateTinyInt

canonicalName() - Method in class org.apache.iceberg.spark.functions.YearsFunction.DateToYearsFunction

canonicalName() - Method in class org.apache.iceberg.spark.functions.YearsFunction.TimestampToYearsFunction

capabilities() - Method in class org.apache.iceberg.spark.RollbackStagedTable

capabilities() - Method in class org.apache.iceberg.spark.source.SparkChangelogTable

capabilities() - Method in class org.apache.iceberg.spark.source.SparkTable

caseSensitive(boolean) - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

caseSensitive() - Method in class org.apache.iceberg.spark.SparkReadConf

caseSensitive(SparkSession) - Static method in class org.apache.iceberg.spark.SparkUtil

caseSensitive() - Method in class org.apache.iceberg.spark.SparkWriteConf

catalog() - Method in class org.apache.iceberg.spark.Spark3Util.CatalogAndIdentifier

catalogAndIdentifier(SparkSession, String) - Static method in class org.apache.iceberg.spark.Spark3Util

catalogAndIdentifier(SparkSession, String, CatalogPlugin) - Static method in class org.apache.iceberg.spark.Spark3Util

catalogAndIdentifier(String, SparkSession, String) - Static method in class org.apache.iceberg.spark.Spark3Util

catalogAndIdentifier(String, SparkSession, String, CatalogPlugin) - Static method in class org.apache.iceberg.spark.Spark3Util

catalogAndIdentifier(SparkSession, List<String>) - Static method in class org.apache.iceberg.spark.Spark3Util

catalogAndIdentifier(SparkSession, List<String>, CatalogPlugin) - Static method in class org.apache.iceberg.spark.Spark3Util

A modified version of Spark's LookupCatalog.CatalogAndIdentifier.unapply Attempts to find the catalog and identifier a multipart identifier represents

CatalogAndIdentifier(CatalogPlugin, Identifier) - Constructor for class org.apache.iceberg.spark.Spark3Util.CatalogAndIdentifier

CatalogAndIdentifier(Pair<CatalogPlugin, Identifier>) - Constructor for class org.apache.iceberg.spark.Spark3Util.CatalogAndIdentifier

catalogAndIdentifier(List<String>, Function<String, C>, BiFunction<String[], String, T>, C, String[]) - Static method in class org.apache.iceberg.spark.SparkUtil

A modified version of Spark's LookupCatalog.CatalogAndIdentifier.unapply Attempts to find the catalog and identifier a multipart identifier represents

ChangelogIterator - Class in org.apache.iceberg.spark

An iterator that transforms rows from changelog tables within a single Spark task.

ChangelogIterator(Iterator<Row>, StructType) - Constructor for class org.apache.iceberg.spark.ChangelogIterator

changeType(Row) - Method in class org.apache.iceberg.spark.ChangelogIterator

changeTypeIndex() - Method in class org.apache.iceberg.spark.ChangelogIterator

CHECK_NULLABILITY - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

CHECK_NULLABILITY - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

CHECK_NULLABILITY_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

CHECK_ORDERING - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

CHECK_ORDERING - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

CHECK_ORDERING_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

checkNullability() - Method in class org.apache.iceberg.spark.SparkWriteConf

checkOrdering() - Method in class org.apache.iceberg.spark.SparkWriteConf

checkSourceCatalog(CatalogPlugin) - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

checkSourceCatalog(CatalogPlugin) - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

close() - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

close() - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

close() - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

ColumnarBatchReader - Class in org.apache.iceberg.spark.data.vectorized

VectorizedReader that returns Spark's ColumnarBatch to support Spark's vectorized read path.

ColumnarBatchReader(List<VectorizedReader<?>>) - Constructor for class org.apache.iceberg.spark.data.vectorized.ColumnarBatchReader

columnSizes() - Method in class org.apache.iceberg.spark.SparkDataFile

ColumnVectorWithFilter - Class in org.apache.iceberg.spark.data.vectorized

ColumnVectorWithFilter(VectorHolder, int[]) - Constructor for class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

commit(Offset) - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

CommitMetadata - Class in org.apache.iceberg.spark

utility class to accept thread local commit properties

commitProperties() - Static method in class org.apache.iceberg.spark.CommitMetadata

commitStagedChanges() - Method in class org.apache.iceberg.spark.RollbackStagedTable

commitStagedChanges() - Method in class org.apache.iceberg.spark.source.StagedSparkTable

compareToFileList(Dataset<Row>) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

ComputeUpdateIterator - Class in org.apache.iceberg.spark

An iterator that finds delete/insert rows which represent an update, and converts them into update records from changelog tables within a single Spark task.

computeUpdates(Iterator<Row>, StructType, String[]) - Static method in class org.apache.iceberg.spark.ChangelogIterator

Creates an iterator composing RemoveCarryoverIterator and ComputeUpdateIterator to remove carry-over rows and compute update rows

contains(String) - Method in class org.apache.iceberg.spark.SparkTableCache

content() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

convert(AggregateFunc) - Static method in class org.apache.iceberg.spark.SparkAggregates

convert(SortOrder) - Static method in class org.apache.iceberg.spark.SparkDistributionAndOrderingUtil

convert(Filter[]) - Static method in class org.apache.iceberg.spark.SparkFilters

convert(Filter) - Static method in class org.apache.iceberg.spark.SparkFilters

convert(Schema) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Schema to a Spark type.

convert(Type) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Type to a Spark type.

convert(StructType) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Spark struct to a Schema with new field ids.

convert(StructType, boolean) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Spark struct to a Schema with new field ids.

convert(DataType) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Spark struct to a Type with new field ids.

convert(Schema, StructType) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Spark struct to a Schema based on the given schema.

convert(Schema, StructType, boolean) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Spark struct to a Schema based on the given schema.

convert(Predicate) - Static method in class org.apache.iceberg.spark.SparkV2Filters

convert(Schema, Row) - Static method in class org.apache.iceberg.spark.SparkValueConverter

convert(Type, Object) - Static method in class org.apache.iceberg.spark.SparkValueConverter

convertToSpark(Type, Object) - Static method in class org.apache.iceberg.spark.SparkValueConverter

convertWithFreshIds(Schema, StructType) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Spark struct to a Schema based on the given schema.

convertWithFreshIds(Schema, StructType, boolean) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Convert a Spark struct to a Schema based on the given schema.

copy() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

copy() - Method in class org.apache.iceberg.spark.actions.SetAccumulator

copy() - Method in class org.apache.iceberg.spark.SparkDataFile

copyOf(Table) - Static method in class org.apache.iceberg.spark.source.SerializableTableWithSize

copyOnWriteMergeDistributionMode() - Method in class org.apache.iceberg.spark.SparkWriteConf

copyWithBranch(String) - Method in class org.apache.iceberg.spark.source.SparkTable

copyWithoutStats() - Method in class org.apache.iceberg.spark.SparkDataFile

copyWithSnapshotId(long) - Method in class org.apache.iceberg.spark.source.SparkTable

createBatchWriterFactory(PhysicalWriteInfo) - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaBatchWrite

CreateChangelogViewProcedure - Class in org.apache.iceberg.spark.procedures

A procedure that creates a view for changed rows.

createNamespace(String[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkCatalog

createNamespace(String[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

createReaderFactory() - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

createTable(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

createTable(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkCatalog

createTable(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

createWriter(int, long) - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaWriterFactory

currentPath() - Method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

D

dataFileFormat() - Method in class org.apache.iceberg.spark.SparkWriteConf

dataType() - Method in class org.apache.iceberg.spark.source.SparkMetadataColumn

dataType() - Method in interface org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter

Returns the type of this parameter.

DateToDaysFunction() - Constructor for class org.apache.iceberg.spark.functions.DaysFunction.DateToDaysFunction

DateToMonthsFunction() - Constructor for class org.apache.iceberg.spark.functions.MonthsFunction.DateToMonthsFunction

DateToYearsFunction() - Constructor for class org.apache.iceberg.spark.functions.YearsFunction.DateToYearsFunction

DaysFunction - Class in org.apache.iceberg.spark.functions

A Spark function implementation for the Iceberg day transform.

DaysFunction() - Constructor for class org.apache.iceberg.spark.functions.DaysFunction

DaysFunction.DateToDaysFunction - Class in org.apache.iceberg.spark.functions

DaysFunction.TimestampToDaysFunction - Class in org.apache.iceberg.spark.functions

decimals(int, int) - Static method in class org.apache.iceberg.spark.data.SparkOrcValueReaders

defaultNamespace() - Method in class org.apache.iceberg.spark.SparkCatalog

defaultNamespace() - Method in class org.apache.iceberg.spark.SparkSessionCatalog

DELETE - Static variable in class org.apache.iceberg.spark.ChangelogIterator

delete(T, T) - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaWriter

Passes information for a row that must be deleted.

DELETE_FORMAT - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

DeletedColumnVector - Class in org.apache.iceberg.spark.data.vectorized

DeletedColumnVector(Type, boolean[]) - Constructor for class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

deletedFilesCount() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

deleteDistributionMode() - Method in class org.apache.iceberg.spark.SparkWriteConf

deletedRowsCount() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

deleteFileFormat() - Method in class org.apache.iceberg.spark.SparkWriteConf

deleteOrphanFiles(Table) - Method in class org.apache.iceberg.spark.actions.SparkActions

DeleteOrphanFilesSparkAction - Class in org.apache.iceberg.spark.actions

An action that removes orphan metadata, data and delete files by listing a given location and comparing the actual files in that location with content and metadata files referenced by all valid snapshots.

DeleteOrphanFilesSparkAction.FileURI - Class in org.apache.iceberg.spark.actions

deleteReachableFiles(String) - Method in class org.apache.iceberg.spark.actions.SparkActions

DeleteReachableFilesSparkAction - Class in org.apache.iceberg.spark.actions

An implementation of DeleteReachableFiles that uses metadata tables in Spark to determine which files should be deleted.

deleteWhere(Filter[]) - Method in class org.apache.iceberg.spark.RollbackStagedTable

deleteWhere(Filter[]) - Method in class org.apache.iceberg.spark.source.SparkTable

deleteWith(Consumer<String>) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

deleteWith(Consumer<String>) - Method in class org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

deleteWith(Consumer<String>) - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

DeltaBatchWrite - Interface in org.apache.spark.sql.connector.iceberg.write

An interface that defines how to write a delta of rows during batch processing.

DeltaWrite - Interface in org.apache.spark.sql.connector.iceberg.write

A logical representation of a data source write that handles a delta of rows.

DeltaWriteBuilder - Interface in org.apache.spark.sql.connector.iceberg.write

An interface for building delta writes.

DeltaWriter<T> - Interface in org.apache.spark.sql.connector.iceberg.write

A data writer responsible for writing a delta of rows.

DeltaWriterFactory - Interface in org.apache.spark.sql.connector.iceberg.write

A factory for creating and initializing delta writers at the executor side.

describe(List<Expression>) - Static method in class org.apache.iceberg.spark.Spark3Util

describe(Expression) - Static method in class org.apache.iceberg.spark.Spark3Util

describe(Schema) - Static method in class org.apache.iceberg.spark.Spark3Util

describe(Type) - Static method in class org.apache.iceberg.spark.Spark3Util

describe(SortOrder) - Static method in class org.apache.iceberg.spark.Spark3Util

description() - Method in class org.apache.iceberg.spark.functions.BucketFunction

description() - Method in class org.apache.iceberg.spark.functions.DaysFunction

description() - Method in class org.apache.iceberg.spark.functions.HoursFunction

description() - Method in class org.apache.iceberg.spark.functions.IcebergVersionFunction

description() - Method in class org.apache.iceberg.spark.functions.MonthsFunction

description() - Method in class org.apache.iceberg.spark.functions.TruncateFunction

description() - Method in class org.apache.iceberg.spark.functions.YearsFunction

description() - Method in class org.apache.iceberg.spark.JobGroupInfo

description() - Method in class org.apache.iceberg.spark.procedures.AncestorsOfProcedure

description() - Method in class org.apache.iceberg.spark.procedures.CreateChangelogViewProcedure

description() - Method in class org.apache.iceberg.spark.procedures.ExpireSnapshotsProcedure

description() - Method in class org.apache.iceberg.spark.procedures.FastForwardBranchProcedure

description() - Method in class org.apache.iceberg.spark.procedures.RemoveOrphanFilesProcedure

description() - Method in class org.apache.iceberg.spark.procedures.RewritePositionDeleteFilesProcedure

description() - Method in class org.apache.iceberg.spark.source.metrics.NumDeletes

description() - Method in class org.apache.iceberg.spark.source.metrics.NumSplits

description() - Method in interface org.apache.spark.sql.connector.iceberg.catalog.Procedure

Returns the description of this procedure.

deserializeOffset(String) - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

destCatalog() - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

destCatalog() - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

destTableIdent() - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

destTableIdent() - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

destTableProps() - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

destTableProps() - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

determineWriteBranch(SparkSession, String) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Determine the write branch.

direction() - Method in class org.apache.iceberg.spark.ExtendedParser.RawOrderField

DISPLAY_STRING - Static variable in class org.apache.iceberg.spark.source.metrics.NumDeletes

DISTRIBUTION_MODE - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

DISTRIBUTION_MODE - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

distributionMode() - Method in class org.apache.iceberg.spark.SparkWriteConf

doBind(DataType) - Method in class org.apache.iceberg.spark.functions.DaysFunction

doBind(DataType) - Method in class org.apache.iceberg.spark.functions.HoursFunction

doBind(DataType) - Method in class org.apache.iceberg.spark.functions.MonthsFunction

doBind(DataType) - Method in class org.apache.iceberg.spark.functions.YearsFunction

dropBackup() - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

dropNamespace(String[], boolean) - Method in class org.apache.iceberg.spark.SparkCatalog

dropNamespace(String[], boolean) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

dropTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

dropTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCatalog

dropTable(Identifier) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

E

ENCODER - Static variable in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

ENCODER - Static variable in class org.apache.iceberg.spark.actions.FileInfo

ENCODER - Static variable in class org.apache.iceberg.spark.actions.ManifestFileBean

END_SNAPSHOT_ID - Static variable in class org.apache.iceberg.spark.SparkReadOptions

END_TIMESTAMP - Static variable in class org.apache.iceberg.spark.SparkReadOptions

endSnapshotId() - Method in class org.apache.iceberg.spark.SparkReadConf

endTimestamp() - Method in class org.apache.iceberg.spark.SparkReadConf

equalAuthorities(Map<String, String>) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

EqualityDeleteRowReader - Class in org.apache.iceberg.spark.source

EqualityDeleteRowReader(CombinedScanTask, Table, Schema, Schema, boolean) - Constructor for class org.apache.iceberg.spark.source.EqualityDeleteRowReader

equals(Object) - Method in class org.apache.iceberg.spark.source.SparkTable

equals(Object) - Method in class org.apache.iceberg.spark.SparkTableUtil.SparkPartition

equalSchemes(Map<String, String>) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

estimatedSize() - Method in class org.apache.iceberg.spark.source.SerializableTableWithSize

estimatedSize() - Method in class org.apache.iceberg.spark.source.SerializableTableWithSize.SerializableMetadataTableWithSize

estimateSize(StructType, long) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Estimate approximate table size based on Spark schema and total records.

estimateStatistics() - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

execute() - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

execute() - Method in class org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

execute() - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

execute() - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

execute() - Method in class org.apache.iceberg.spark.actions.RewriteDataFilesSparkAction

execute() - Method in class org.apache.iceberg.spark.actions.RewriteManifestsSparkAction

execute() - Method in class org.apache.iceberg.spark.actions.RewritePositionDeleteFilesSparkAction

execute() - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

executeDeleteWith(ExecutorService) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

executeDeleteWith(ExecutorService) - Method in class org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

executeDeleteWith(ExecutorService) - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

existingFilesCount() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

existingRowsCount() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

expireFiles() - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

Expires snapshots and commits the changes to the table, returning a Dataset of files to delete.

expireOlderThan(long) - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

expireSnapshotId(long) - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

expireSnapshots(Table) - Method in class org.apache.iceberg.spark.actions.SparkActions

ExpireSnapshotsProcedure - Class in org.apache.iceberg.spark.procedures

A procedure that expires snapshots in a table.

ExpireSnapshotsSparkAction - Class in org.apache.iceberg.spark.actions

An action that performs the same operation as ExpireSnapshots but uses Spark to determine the delta in files between the pre and post-expiration table metadata.

ExtendedLogicalWriteInfo - Interface in org.apache.spark.sql.connector.iceberg.write

A class that holds logical write information not covered by LogicalWriteInfo in Spark.

ExtendedParser - Interface in org.apache.iceberg.spark

ExtendedParser.RawOrderField - Class in org.apache.iceberg.spark

extensionsEnabled(SparkSession) - Static method in class org.apache.iceberg.spark.Spark3Util

extractCatalog(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.IcebergSource

extractIdentifier(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.IcebergSource

extractTimeTravelTimestamp(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.IcebergSource

extractTimeTravelVersion(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.IcebergSource

extraSnapshotMetadata() - Method in class org.apache.iceberg.spark.SparkWriteConf

F

FANOUT_ENABLED - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

fanoutWriterEnabled() - Method in class org.apache.iceberg.spark.SparkWriteConf

FastForwardBranchProcedure - Class in org.apache.iceberg.spark.procedures

fetchSetIds(Table) - Method in class org.apache.iceberg.spark.ScanTaskSetManager

fetchTasks(Table, String) - Method in class org.apache.iceberg.spark.ScanTaskSetManager

field(Types.NestedField, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithoutReordering

field(Types.NestedField, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithReordering

field(Types.NestedField, String) - Method in class org.apache.iceberg.spark.Spark3Util.DescribeSchemaVisitor

fieldNameAndType(DataType, int) - Method in class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

FILE_OPEN_COST - Static variable in class org.apache.iceberg.spark.SparkReadOptions

FileInfo - Class in org.apache.iceberg.spark.actions

FileInfo(String, String) - Constructor for class org.apache.iceberg.spark.actions.FileInfo

FileInfo() - Constructor for class org.apache.iceberg.spark.actions.FileInfo

FileRewriteCoordinator - Class in org.apache.iceberg.spark

fileSizeInBytes() - Method in class org.apache.iceberg.spark.SparkDataFile

FileURI(String, String, String, String) - Constructor for class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

FileURI() - Constructor for class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

filter(Expression) - Method in class org.apache.iceberg.spark.actions.RewriteDataFilesSparkAction

filter(Expression) - Method in class org.apache.iceberg.spark.actions.RewritePositionDeleteFilesSparkAction

filterPartitions(List<SparkTableUtil.SparkPartition>, Map<String, String>) - Static method in class org.apache.iceberg.spark.SparkTableUtil

format() - Method in class org.apache.iceberg.spark.SparkDataFile

G

generateIndicesToIdentifySameRow(int, Set<Integer>) - Static method in class org.apache.iceberg.spark.ChangelogIterator

get(SparkSession) - Static method in class org.apache.iceberg.spark.actions.SparkActions

get() - Static method in class org.apache.iceberg.spark.actions.SparkActions

get() - Static method in class org.apache.iceberg.spark.FileRewriteCoordinator

get() - Static method in class org.apache.iceberg.spark.PositionDeletesRewriteCoordinator

get() - Static method in class org.apache.iceberg.spark.ScanTaskSetManager

get(int, Class<T>) - Method in class org.apache.iceberg.spark.SparkStructLike

get() - Static method in class org.apache.iceberg.spark.SparkTableCache

get(String) - Method in class org.apache.iceberg.spark.SparkTableCache

getAddedSnapshotId() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

getArray(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getArray(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getArray(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getArray(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getAuthority() - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

getBinary(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getBinary(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getBinary(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getBinary(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getBoolean(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getBoolean(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getBoolean(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getBoolean(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getByte(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getByte(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getByte(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getChild(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getChild(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getChild(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getContent() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

getDecimal(int, int, int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getDecimal(int, int, int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getDecimal(int, int, int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getDecimal(int, int, int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getDefaultReadLimit() - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

getDouble(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getDouble(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getDouble(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getDouble(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getFloat(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getFloat(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getFloat(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getFloat(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getFormat() - Method in class org.apache.iceberg.spark.SparkTableUtil.SparkPartition

getInt(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getInt(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getInt(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getInt(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getJobGroupInfo(SparkContext) - Static method in class org.apache.iceberg.spark.JobGroupUtils

getLength() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

getLong(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getLong(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getLong(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getLong(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getMap(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getMap(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getMap(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getPartitions(SparkSession, Path, String, Map<String, String>) - Static method in class org.apache.iceberg.spark.Spark3Util

Deprecated.

use Spark3Util.getPartitions(SparkSession, Path, String, Map, PartitionSpec)

getPartitions(SparkSession, Path, String, Map<String, String>, PartitionSpec) - Static method in class org.apache.iceberg.spark.Spark3Util

Use Spark to list all partitions in the table.

getPartitions(SparkSession, String) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Returns all partitions in the table.

getPartitions(SparkSession, TableIdentifier, Map<String, String>) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Returns all partitions in the table.

getPartitionsByFilter(SparkSession, String, String) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Returns partitions that match the specified 'predicate'.

getPartitionsByFilter(SparkSession, TableIdentifier, Expression) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Returns partitions that match the specified 'predicate'.

getPartitionSpecId() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

getPath() - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

getPath() - Method in class org.apache.iceberg.spark.actions.FileInfo

getPath() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

getScheme() - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

getShort(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getShort(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getShort(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getTable(StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.source.IcebergSource

getType() - Method in class org.apache.iceberg.spark.actions.FileInfo

getUri() - Method in class org.apache.iceberg.spark.SparkTableUtil.SparkPartition

getUriAsString() - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

getUTF8String(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

getUTF8String(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

getUTF8String(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

getUTF8String(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

getValues() - Method in class org.apache.iceberg.spark.SparkTableUtil.SparkPartition

groupId() - Method in class org.apache.iceberg.spark.JobGroupInfo

H

hadoopConfCatalogOverrides(SparkSession, String) - Static method in class org.apache.iceberg.spark.SparkUtil

Pulls any Catalog specific overrides for the Hadoop conf from the current SparkSession, which can be set via `spark.sql.catalog.$catalogName.hadoop.*`

HANDLE_TIMESTAMP_WITHOUT_TIMEZONE - Static variable in class org.apache.iceberg.spark.SparkReadOptions

HANDLE_TIMESTAMP_WITHOUT_TIMEZONE - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

HANDLE_TIMESTAMP_WITHOUT_TIMEZONE - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

HANDLE_TIMESTAMP_WITHOUT_TIMEZONE_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

handleTimestampWithoutZone() - Method in class org.apache.iceberg.spark.SparkReadConf

Enables reading a timestamp without time zone as a timestamp with time zone.

handleTimestampWithoutZone() - Method in class org.apache.iceberg.spark.SparkWriteConf

Enables writing a timestamp with time zone as a timestamp without time zone.

hash(ByteBuffer) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBinary

hash(BigDecimal) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketDecimal

hash(int) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketInt

hash(long) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketLong

hash(String) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketString

hashCode() - Method in class org.apache.iceberg.spark.source.SparkTable

hashCode() - Method in class org.apache.iceberg.spark.SparkTableUtil.SparkPartition

HasIcebergCatalog - Interface in org.apache.iceberg.spark.source

hasNext() - Method in class org.apache.iceberg.spark.ComputeUpdateIterator

hasNext() - Method in class org.apache.iceberg.spark.RemoveNetCarryoverIterator

hasNull() - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

hasNull() - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

hasNull() - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

hasTimestampWithoutZone(Schema) - Static method in class org.apache.iceberg.spark.SparkUtil

Responsible for checking if the table schema has a timestamp without timezone column

HoursFunction - Class in org.apache.iceberg.spark.functions

A Spark function implementation for the Iceberg hour transform.

HoursFunction() - Constructor for class org.apache.iceberg.spark.functions.HoursFunction

HoursFunction.TimestampToHoursFunction - Class in org.apache.iceberg.spark.functions

I

IcebergArrowColumnVector - Class in org.apache.iceberg.spark.data.vectorized

Implementation of Spark's ColumnVector interface.

IcebergArrowColumnVector(VectorHolder) - Constructor for class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

icebergCatalog() - Method in interface org.apache.iceberg.spark.source.HasIcebergCatalog

Returns the underlying Catalog backing this Spark Catalog

icebergCatalog() - Method in class org.apache.iceberg.spark.SparkCatalog

icebergCatalog() - Method in class org.apache.iceberg.spark.SparkSessionCatalog

IcebergSource - Class in org.apache.iceberg.spark.source

The IcebergSource loads/writes tables with format "iceberg".

IcebergSource() - Constructor for class org.apache.iceberg.spark.source.IcebergSource

IcebergSpark - Class in org.apache.iceberg.spark

IcebergVersionFunction - Class in org.apache.iceberg.spark.functions

A function for use in SQL that returns the current Iceberg version, e.g.

IcebergVersionFunction() - Constructor for class org.apache.iceberg.spark.functions.IcebergVersionFunction

identifier() - Method in class org.apache.iceberg.spark.Spark3Util.CatalogAndIdentifier

identifierToTableIdentifier(Identifier) - Static method in class org.apache.iceberg.spark.Spark3Util

importSparkPartitions(SparkSession, List<SparkTableUtil.SparkPartition>, Table, PartitionSpec, String, boolean) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Import files from given partitions to an Iceberg table.

importSparkPartitions(SparkSession, List<SparkTableUtil.SparkPartition>, Table, PartitionSpec, String) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Import files from given partitions to an Iceberg table.

importSparkTable(SparkSession, TableIdentifier, Table, String, Map<String, String>, boolean) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Import files from an existing Spark table to an Iceberg table.

importSparkTable(SparkSession, TableIdentifier, Table, String, boolean) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Import files from an existing Spark table to an Iceberg table.

importSparkTable(SparkSession, TableIdentifier, Table, String) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Import files from an existing Spark table to an Iceberg table.

indexQuotedNameById(Schema) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

inferPartitioning(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.IcebergSource

inferSchema(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.IcebergSource

initialize(String, CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

initialize(String, CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.SparkCatalog

initialize(String, CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

initialOffset() - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

inputTypes() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBinary

inputTypes() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketDecimal

inputTypes() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketInt

inputTypes() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketLong

inputTypes() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketString

inputTypes() - Method in class org.apache.iceberg.spark.functions.DaysFunction.DateToDaysFunction

inputTypes() - Method in class org.apache.iceberg.spark.functions.DaysFunction.TimestampToDaysFunction

inputTypes() - Method in class org.apache.iceberg.spark.functions.HoursFunction.TimestampToHoursFunction

inputTypes() - Method in class org.apache.iceberg.spark.functions.MonthsFunction.DateToMonthsFunction

inputTypes() - Method in class org.apache.iceberg.spark.functions.MonthsFunction.TimestampToMonthsFunction

inputTypes() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBigInt

inputTypes() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBinary

inputTypes() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateDecimal

inputTypes() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateInt

inputTypes() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateSmallInt

inputTypes() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateString

inputTypes() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateTinyInt

inputTypes() - Method in class org.apache.iceberg.spark.functions.YearsFunction.DateToYearsFunction

inputTypes() - Method in class org.apache.iceberg.spark.functions.YearsFunction.TimestampToYearsFunction

INSERT - Static variable in class org.apache.iceberg.spark.ChangelogIterator

insert(T) - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaWriter

Passes a row to insert.

interruptOnCancel() - Method in class org.apache.iceberg.spark.JobGroupInfo

invalidateTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

invalidateTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCatalog

invalidateTable(Identifier) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

invoke(int, byte[]) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBinary

invoke(int, Decimal) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketDecimal

invoke(int, int) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketInt

invoke(int, long) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketLong

invoke(int, UTF8String) - Static method in class org.apache.iceberg.spark.functions.BucketFunction.BucketString

invoke(int) - Static method in class org.apache.iceberg.spark.functions.DaysFunction.DateToDaysFunction

invoke(long) - Static method in class org.apache.iceberg.spark.functions.DaysFunction.TimestampToDaysFunction

invoke(long) - Static method in class org.apache.iceberg.spark.functions.HoursFunction.TimestampToHoursFunction

invoke(int) - Static method in class org.apache.iceberg.spark.functions.MonthsFunction.DateToMonthsFunction

invoke(long) - Static method in class org.apache.iceberg.spark.functions.MonthsFunction.TimestampToMonthsFunction

invoke(int, long) - Static method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBigInt

invoke(int, byte[]) - Static method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBinary

invoke(int, Decimal) - Static method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateDecimal

invoke(int, int) - Static method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateInt

invoke(int, short) - Static method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateSmallInt

invoke(int, UTF8String) - Static method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateString

invoke(int, byte) - Static method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateTinyInt

invoke(int) - Static method in class org.apache.iceberg.spark.functions.YearsFunction.DateToYearsFunction

invoke(long) - Static method in class org.apache.iceberg.spark.functions.YearsFunction.TimestampToYearsFunction

io(FileIO) - Method in class org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

isDifferentValue(Row, Row, int) - Method in class org.apache.iceberg.spark.ChangelogIterator

isMapType(DataType) - Method in class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

isNullable() - Method in class org.apache.iceberg.spark.source.SparkMetadataColumn

isNullAt(int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnVectorWithFilter

isNullAt(int) - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

isNullAt(int) - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

isNullAt(int) - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

ISOLATION_LEVEL - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

isolationLevel() - Method in class org.apache.iceberg.spark.SparkWriteConf

isSameRecord(Row, Row, int[]) - Method in class org.apache.iceberg.spark.ChangelogIterator

isStringType(DataType) - Method in class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

isZero() - Method in class org.apache.iceberg.spark.actions.SetAccumulator

J

JobGroupInfo - Class in org.apache.iceberg.spark

Captures information about the current job which is used for displaying on the UI

JobGroupInfo(String, String, boolean) - Constructor for class org.apache.iceberg.spark.JobGroupInfo

JobGroupUtils - Class in org.apache.iceberg.spark

K

keyMetadata() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

keyMetadata() - Method in class org.apache.iceberg.spark.SparkDataFile

L

latestOffset() - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

latestOffset(Offset, ReadLimit) - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

length() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

list(ArrayType, GroupType, T) - Method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

list() - Static method in class org.apache.iceberg.spark.functions.SparkFunctions

list(Types.ListType, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithoutReordering

list(Types.ListType, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithReordering

list(Types.ListType, String) - Method in class org.apache.iceberg.spark.Spark3Util.DescribeSchemaVisitor

listNamespaces() - Method in class org.apache.iceberg.spark.SparkCatalog

listNamespaces(String[]) - Method in class org.apache.iceberg.spark.SparkCatalog

listNamespaces() - Method in class org.apache.iceberg.spark.SparkSessionCatalog

listNamespaces(String[]) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

listPartition(SparkTableUtil.SparkPartition, PartitionSpec, SerializableConfiguration, MetricsConfig) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Deprecated.

use TableMigrationUtil.listPartition(Map, String, String, PartitionSpec, Configuration, MetricsConfig, NameMapping)

listPartition(SparkTableUtil.SparkPartition, PartitionSpec, SerializableConfiguration, MetricsConfig, NameMapping) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Deprecated.

use TableMigrationUtil.listPartition(Map, String, String, PartitionSpec, Configuration, MetricsConfig, NameMapping)

listTables(String[]) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

listTables(String[]) - Method in class org.apache.iceberg.spark.SparkCatalog

listTables(String[]) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

load(String) - Static method in class org.apache.iceberg.spark.functions.SparkFunctions

loadCatalogMetadataTable(SparkSession, Table, MetadataTableType) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Deprecated.

since 0.14.0, will be removed in 0.15.0; use SparkTableUtil.loadMetadataTable(SparkSession, Table, MetadataTableType).

loadFunction(Identifier) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

loadIcebergCatalog(SparkSession, String) - Static method in class org.apache.iceberg.spark.Spark3Util

Returns the underlying Iceberg Catalog object represented by a Spark Catalog

loadIcebergTable(SparkSession, String) - Static method in class org.apache.iceberg.spark.Spark3Util

Returns an Iceberg Table by its name from a Spark V2 Catalog.

loadMetadataTable(SparkSession, Table, MetadataTableType) - Static method in class org.apache.iceberg.spark.SparkTableUtil

loadMetadataTable(SparkSession, Table, MetadataTableType, Map<String, String>) - Static method in class org.apache.iceberg.spark.SparkTableUtil

loadNamespaceMetadata(String[]) - Method in class org.apache.iceberg.spark.SparkCatalog

loadNamespaceMetadata(String[]) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

loadProcedure(Identifier) - Method in interface org.apache.spark.sql.connector.iceberg.catalog.ProcedureCatalog

Load a stored procedure by identifier.

loadTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

loadTable(Identifier, String) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

loadTable(Identifier, long) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

loadTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCatalog

loadTable(Identifier, String) - Method in class org.apache.iceberg.spark.SparkCatalog

loadTable(Identifier, long) - Method in class org.apache.iceberg.spark.SparkCatalog

loadTable(Identifier) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

loadTable(Identifier, String) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

loadTable(Identifier, long) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

LOCALITY - Static variable in class org.apache.iceberg.spark.SparkReadOptions

localityEnabled() - Method in class org.apache.iceberg.spark.SparkReadConf

location(String) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

location() - Method in class org.apache.iceberg.spark.PathIdentifier

LOOKBACK - Static variable in class org.apache.iceberg.spark.SparkReadOptions

lowerBounds() - Method in class org.apache.iceberg.spark.SparkDataFile

M

ManifestFileBean - Class in org.apache.iceberg.spark.actions

ManifestFileBean() - Constructor for class org.apache.iceberg.spark.actions.ManifestFileBean

map(MapType, GroupType, T, T) - Method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

map(Types.MapType, Supplier<Type>, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithoutReordering

map(Types.MapType, Supplier<Type>, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithReordering

map(Types.MapType, String, String) - Method in class org.apache.iceberg.spark.Spark3Util.DescribeSchemaVisitor

mapKeyType(DataType) - Method in class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

mapValueType(DataType) - Method in class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

maxFilesPerMicroBatch() - Method in class org.apache.iceberg.spark.SparkReadConf

maxRecordsPerMicroBatch() - Method in class org.apache.iceberg.spark.SparkReadConf

merge(AccumulatorV2<T, Set<T>>) - Method in class org.apache.iceberg.spark.actions.SetAccumulator

MERGE_SCHEMA - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

MERGE_SCHEMA_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

mergeSchema() - Method in class org.apache.iceberg.spark.SparkWriteConf

message(StructType, MessageType, List<T>) - Method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

metadataColumns() - Method in class org.apache.iceberg.spark.source.SparkChangelogTable

metadataColumns() - Method in class org.apache.iceberg.spark.source.SparkTable

metadataSchema() - Method in interface org.apache.spark.sql.connector.iceberg.write.ExtendedLogicalWriteInfo

The schema of the input metadata from Spark to data source.

metrics() - Method in class org.apache.iceberg.spark.data.SparkAvroWriter

metrics() - Method in class org.apache.iceberg.spark.data.SparkOrcWriter

migrateTable(String) - Method in class org.apache.iceberg.spark.actions.SparkActions

MigrateTableSparkAction - Class in org.apache.iceberg.spark.actions

Takes a Spark table in the source catalog and attempts to transform it into an Iceberg table in the same location with the same identifier.

minSequenceNumber() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

MonthsFunction - Class in org.apache.iceberg.spark.functions

A Spark function implementation for the Iceberg month transform.

MonthsFunction() - Constructor for class org.apache.iceberg.spark.functions.MonthsFunction

MonthsFunction.DateToMonthsFunction - Class in org.apache.iceberg.spark.functions

MonthsFunction.TimestampToMonthsFunction - Class in org.apache.iceberg.spark.functions

N

name() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBase

name() - Method in class org.apache.iceberg.spark.functions.BucketFunction

name() - Method in class org.apache.iceberg.spark.functions.DaysFunction

name() - Method in class org.apache.iceberg.spark.functions.HoursFunction

name() - Method in class org.apache.iceberg.spark.functions.HoursFunction.TimestampToHoursFunction

name() - Method in class org.apache.iceberg.spark.functions.IcebergVersionFunction

name() - Method in class org.apache.iceberg.spark.functions.MonthsFunction

name() - Method in class org.apache.iceberg.spark.functions.TruncateFunction

name() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBase

name() - Method in class org.apache.iceberg.spark.functions.YearsFunction

name() - Method in class org.apache.iceberg.spark.PathIdentifier

name() - Method in class org.apache.iceberg.spark.RollbackStagedTable

name() - Method in class org.apache.iceberg.spark.source.metrics.NumDeletes

name() - Method in class org.apache.iceberg.spark.source.metrics.NumSplits

name() - Method in class org.apache.iceberg.spark.source.metrics.TaskNumDeletes

name() - Method in class org.apache.iceberg.spark.source.metrics.TaskNumSplits

name() - Method in class org.apache.iceberg.spark.source.SparkChangelogTable

name() - Method in class org.apache.iceberg.spark.source.SparkMetadataColumn

name() - Method in class org.apache.iceberg.spark.source.SparkTable

name() - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

name() - Method in class org.apache.iceberg.spark.SparkCatalog

name() - Method in class org.apache.iceberg.spark.SparkSessionCatalog

name() - Method in interface org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter

Returns the name of this parameter.

namespace() - Method in class org.apache.iceberg.spark.PathIdentifier

namespaceExists(String[]) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

nanValueCounts() - Method in class org.apache.iceberg.spark.SparkDataFile

newBuilder(String) - Static method in class org.apache.iceberg.spark.procedures.SparkProcedures

newRowLevelOperationBuilder(RowLevelOperationInfo) - Method in class org.apache.iceberg.spark.source.SparkTable

newScanBuilder(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.RollbackStagedTable

newScanBuilder(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.SparkChangelogTable

newScanBuilder(CaseInsensitiveStringMap) - Method in class org.apache.iceberg.spark.source.SparkTable

newWriteBuilder(LogicalWriteInfo) - Method in class org.apache.iceberg.spark.RollbackStagedTable

newWriteBuilder(LogicalWriteInfo) - Method in class org.apache.iceberg.spark.source.SparkTable

newWriteBuilder(LogicalWriteInfo) - Method in interface org.apache.spark.sql.connector.iceberg.write.SupportsDelta

next() - Method in class org.apache.iceberg.spark.ComputeUpdateIterator

next() - Method in class org.apache.iceberg.spark.RemoveNetCarryoverIterator

NoSuchProcedureException - Exception in org.apache.spark.sql.catalyst.analysis

NoSuchProcedureException(Identifier) - Constructor for exception org.apache.spark.sql.catalyst.analysis.NoSuchProcedureException

nullabilityHolder() - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

nullOrder() - Method in class org.apache.iceberg.spark.ExtendedParser.RawOrderField

nullType() - Method in class org.apache.iceberg.spark.data.AvroWithSparkSchemaVisitor

nullValueCounts() - Method in class org.apache.iceberg.spark.SparkDataFile

NumDeletes - Class in org.apache.iceberg.spark.source.metrics

NumDeletes() - Constructor for class org.apache.iceberg.spark.source.metrics.NumDeletes

numNulls() - Method in class org.apache.iceberg.spark.data.vectorized.DeletedColumnVector

numNulls() - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

numNulls() - Method in class org.apache.iceberg.spark.data.vectorized.RowPositionColumnVector

NumSplits - Class in org.apache.iceberg.spark.source.metrics

NumSplits() - Constructor for class org.apache.iceberg.spark.source.metrics.NumSplits

O

olderThan(long) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

open(FileScanTask) - Method in class org.apache.iceberg.spark.source.EqualityDeleteRowReader

optional(String, DataType) - Static method in interface org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter

Creates an optional input parameter.

orcBatchSize() - Method in class org.apache.iceberg.spark.SparkReadConf

orcVectorizationEnabled() - Method in class org.apache.iceberg.spark.SparkReadConf

org.apache.iceberg.spark - package org.apache.iceberg.spark

org.apache.iceberg.spark.actions - package org.apache.iceberg.spark.actions

org.apache.iceberg.spark.data - package org.apache.iceberg.spark.data

org.apache.iceberg.spark.data.vectorized - package org.apache.iceberg.spark.data.vectorized

org.apache.iceberg.spark.functions - package org.apache.iceberg.spark.functions

org.apache.iceberg.spark.procedures - package org.apache.iceberg.spark.procedures

org.apache.iceberg.spark.source - package org.apache.iceberg.spark.source

org.apache.iceberg.spark.source.metrics - package org.apache.iceberg.spark.source.metrics

org.apache.spark.sql.catalyst.analysis - package org.apache.spark.sql.catalyst.analysis

org.apache.spark.sql.connector.iceberg.catalog - package org.apache.spark.sql.connector.iceberg.catalog

org.apache.spark.sql.connector.iceberg.write - package org.apache.spark.sql.connector.iceberg.write

OUTPUT_SPEC_ID - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

outputSpecId() - Method in class org.apache.iceberg.spark.SparkWriteConf

outputType() - Method in class org.apache.iceberg.spark.procedures.AncestorsOfProcedure

outputType() - Method in class org.apache.iceberg.spark.procedures.CreateChangelogViewProcedure

outputType() - Method in class org.apache.iceberg.spark.procedures.ExpireSnapshotsProcedure

outputType() - Method in class org.apache.iceberg.spark.procedures.FastForwardBranchProcedure

outputType() - Method in class org.apache.iceberg.spark.procedures.RemoveOrphanFilesProcedure

outputType() - Method in class org.apache.iceberg.spark.procedures.RewritePositionDeleteFilesProcedure

outputType() - Method in interface org.apache.spark.sql.connector.iceberg.catalog.Procedure

Returns the type of rows produced by this procedure.

OVERWRITE_MODE - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

overwriteMode() - Method in class org.apache.iceberg.spark.SparkWriteConf

P

parameters() - Method in class org.apache.iceberg.spark.procedures.AncestorsOfProcedure

parameters() - Method in class org.apache.iceberg.spark.procedures.CreateChangelogViewProcedure

parameters() - Method in class org.apache.iceberg.spark.procedures.ExpireSnapshotsProcedure

parameters() - Method in class org.apache.iceberg.spark.procedures.FastForwardBranchProcedure

parameters() - Method in class org.apache.iceberg.spark.procedures.RemoveOrphanFilesProcedure

parameters() - Method in class org.apache.iceberg.spark.procedures.RewritePositionDeleteFilesProcedure

parameters() - Method in interface org.apache.spark.sql.connector.iceberg.catalog.Procedure

Returns the input parameters of this procedure.

parquetBatchSize() - Method in class org.apache.iceberg.spark.SparkReadConf

parquetVectorizationEnabled() - Method in class org.apache.iceberg.spark.SparkReadConf

ParquetWithSparkSchemaVisitor<T> - Class in org.apache.iceberg.spark.data

Visitor for traversing a Parquet type with a companion Spark type.

ParquetWithSparkSchemaVisitor() - Constructor for class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

parseSortOrder(SparkSession, String) - Static method in interface org.apache.iceberg.spark.ExtendedParser

parseSortOrder(String) - Method in interface org.apache.iceberg.spark.ExtendedParser

partition(InternalRow) - Method in class org.apache.iceberg.spark.source.SparkPartitionedFanoutWriter

partition(InternalRow) - Method in class org.apache.iceberg.spark.source.SparkPartitionedWriter

partition() - Method in class org.apache.iceberg.spark.SparkDataFile

partitionDF(SparkSession, String) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Returns a DataFrame with a row for each partition in the table.

partitionDFByFilter(SparkSession, String, String) - Static method in class org.apache.iceberg.spark.SparkTableUtil

Returns a DataFrame with a row for each partition that matches the specified 'expression'.

partitioning() - Method in class org.apache.iceberg.spark.RollbackStagedTable

partitioning() - Method in class org.apache.iceberg.spark.source.SparkTable

partitionMapToExpression(StructType, Map<String, String>) - Static method in class org.apache.iceberg.spark.SparkUtil

Get a List of Spark filter Expression.

partitions() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

partitionSpecId() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

path() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

path(String) - Method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

path() - Method in class org.apache.iceberg.spark.SparkDataFile

PATH - Static variable in class org.apache.iceberg.spark.SparkReadOptions

PathIdentifier - Class in org.apache.iceberg.spark

PathIdentifier(String) - Constructor for class org.apache.iceberg.spark.PathIdentifier

planInputPartitions(Offset, Offset) - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

pos() - Method in class org.apache.iceberg.spark.SparkDataFile

PositionDeletesRewriteCoordinator - Class in org.apache.iceberg.spark

positionDeltaMergeDistributionMode() - Method in class org.apache.iceberg.spark.SparkWriteConf

prefixMismatchMode(DeleteOrphanFiles.PrefixMismatchMode) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

PRESERVE_DATA_GROUPING - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

PRESERVE_DATA_GROUPING_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

preserveDataGrouping() - Method in class org.apache.iceberg.spark.SparkReadConf

primitive(DataType, PrimitiveType) - Method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

primitive(Type.PrimitiveType) - Method in class org.apache.iceberg.spark.PruneColumnsWithoutReordering

primitive(Type.PrimitiveType) - Method in class org.apache.iceberg.spark.PruneColumnsWithReordering

primitive(Type.PrimitiveType) - Method in class org.apache.iceberg.spark.Spark3Util.DescribeSchemaVisitor

Procedure - Interface in org.apache.spark.sql.connector.iceberg.catalog

An interface representing a stored procedure available for execution.

ProcedureCatalog - Interface in org.apache.spark.sql.connector.iceberg.catalog

A catalog API for working with stored procedures.

ProcedureParameter - Interface in org.apache.spark.sql.connector.iceberg.catalog

An input parameter of a stored procedure.

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBinary

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketDecimal

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketInt

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketLong

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketString

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.DaysFunction.DateToDaysFunction

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.DaysFunction.TimestampToDaysFunction

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.HoursFunction.TimestampToHoursFunction

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.MonthsFunction.DateToMonthsFunction

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.MonthsFunction.TimestampToMonthsFunction

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBigInt

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBinary

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateDecimal

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateInt

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateSmallInt

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateString

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateTinyInt

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.YearsFunction.DateToYearsFunction

produceResult(InternalRow) - Method in class org.apache.iceberg.spark.functions.YearsFunction.TimestampToYearsFunction

properties() - Method in class org.apache.iceberg.spark.RollbackStagedTable

properties() - Method in class org.apache.iceberg.spark.source.SparkTable

prune(Schema, StructType) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Prune columns from a Schema using a Spark type projection.

prune(Schema, StructType, List<Expression>) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Prune columns from a Schema using a Spark type projection.

prune(Schema, StructType, Expression, boolean) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Prune columns from a Schema using a Spark type projection.

pruneColumns(StructType) - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

PruneColumnsWithoutReordering - Class in org.apache.iceberg.spark

PruneColumnsWithReordering - Class in org.apache.iceberg.spark

purgeTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

purgeTable(Identifier) - Method in class org.apache.iceberg.spark.SparkCatalog

purgeTable(Identifier) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

pushAggregation(Aggregation) - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

pushedFilters() - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

pushFilters(Filter[]) - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

Q

quotedFullIdentifier(String, Identifier) - Static method in class org.apache.iceberg.spark.Spark3Util

R

RawOrderField(Term, SortDirection, NullOrder) - Constructor for class org.apache.iceberg.spark.ExtendedParser.RawOrderField

read(InternalRow, Decoder) - Method in class org.apache.iceberg.spark.data.SparkAvroReader

read(VectorizedRowBatch, int) - Method in class org.apache.iceberg.spark.data.SparkOrcReader

read(ColumnarBatch, int) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnarBatchReader

readSchema() - Method in class org.apache.iceberg.spark.source.SparkScanBuilder

rebuildCreateProperties(Map<String, String>) - Static method in class org.apache.iceberg.spark.Spark3Util

recordCount() - Method in class org.apache.iceberg.spark.SparkDataFile

registerBucketUDF(SparkSession, String, DataType, int) - Static method in class org.apache.iceberg.spark.IcebergSpark

registerTruncateUDF(SparkSession, String, DataType, int) - Static method in class org.apache.iceberg.spark.IcebergSpark

remove(String) - Method in class org.apache.iceberg.spark.SparkTableCache

removeCarryovers(Iterator<Row>, StructType) - Static method in class org.apache.iceberg.spark.ChangelogIterator

Creates an iterator that removes carry-over rows from a changelog table.

RemoveNetCarryoverIterator - Class in org.apache.iceberg.spark

This class computes the net changes across multiple snapshots.

RemoveNetCarryoverIterator(Iterator<Row>, StructType) - Constructor for class org.apache.iceberg.spark.RemoveNetCarryoverIterator

removeNetCarryovers(Iterator<Row>, StructType) - Static method in class org.apache.iceberg.spark.ChangelogIterator

RemoveOrphanFilesProcedure - Class in org.apache.iceberg.spark.procedures

A procedure that removes orphan files in a table.

removeTasks(Table, String) - Method in class org.apache.iceberg.spark.ScanTaskSetManager

renameTable(Identifier, Identifier) - Method in class org.apache.iceberg.spark.SparkCachedTableCatalog

renameTable(Identifier, Identifier) - Method in class org.apache.iceberg.spark.SparkCatalog

renameTable(Identifier, Identifier) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

required(String, DataType) - Static method in interface org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter

Creates a required input parameter.

required() - Method in interface org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter

Returns true if this parameter is required.

reset() - Method in class org.apache.iceberg.spark.actions.SetAccumulator

resultType() - Method in class org.apache.iceberg.spark.functions.BucketFunction.BucketBase

resultType() - Method in class org.apache.iceberg.spark.functions.HoursFunction.TimestampToHoursFunction

resultType() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBigInt

resultType() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBinary

resultType() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateDecimal

resultType() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateInt

resultType() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateSmallInt

resultType() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateString

resultType() - Method in class org.apache.iceberg.spark.functions.TruncateFunction.TruncateTinyInt

retainLast(int) - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

rewriteDataFiles(Table) - Method in class org.apache.iceberg.spark.actions.SparkActions

RewriteDataFilesSparkAction - Class in org.apache.iceberg.spark.actions

rewriteIf(Predicate<ManifestFile>) - Method in class org.apache.iceberg.spark.actions.RewriteManifestsSparkAction

rewriteManifests(Table) - Method in class org.apache.iceberg.spark.actions.SparkActions

RewriteManifestsSparkAction - Class in org.apache.iceberg.spark.actions

An action that rewrites manifests in a distributed manner and co-locates metadata for partitions.

RewritePositionDeleteFilesProcedure - Class in org.apache.iceberg.spark.procedures

A procedure that rewrites position delete files in a table.

RewritePositionDeleteFilesSparkAction - Class in org.apache.iceberg.spark.actions

Spark implementation of RewritePositionDeleteFiles.

rewritePositionDeletes(Table) - Method in class org.apache.iceberg.spark.actions.SparkActions

REWRITTEN_FILE_SCAN_TASK_SET_ID - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

rewrittenFileSetId() - Method in class org.apache.iceberg.spark.SparkWriteConf

RollbackStagedTable - Class in org.apache.iceberg.spark

An implementation of StagedTable that mimics the behavior of Spark's non-atomic CTAS and RTAS.

RollbackStagedTable(TableCatalog, Identifier, Table) - Constructor for class org.apache.iceberg.spark.RollbackStagedTable

rowId() - Method in interface org.apache.spark.sql.connector.iceberg.write.SupportsDelta

Returns the row ID column references that should be used for row equality.

rowIdSchema() - Method in interface org.apache.spark.sql.connector.iceberg.write.ExtendedLogicalWriteInfo

The schema of the ID columns from Spark to data source.

rowIterator() - Method in class org.apache.iceberg.spark.ChangelogIterator

RowPositionColumnVector - Class in org.apache.iceberg.spark.data.vectorized

rowType() - Method in class org.apache.iceberg.spark.ChangelogIterator

S

SCAN_TASK_SET_ID - Static variable in class org.apache.iceberg.spark.SparkReadOptions

scanTaskSetId() - Method in class org.apache.iceberg.spark.SparkReadConf

ScanTaskSetManager - Class in org.apache.iceberg.spark

schema(Schema, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithoutReordering

schema(Schema, Supplier<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithReordering

schema() - Method in class org.apache.iceberg.spark.RollbackStagedTable

schema() - Method in class org.apache.iceberg.spark.source.SparkChangelogTable

schema() - Method in class org.apache.iceberg.spark.source.SparkTable

schema(Schema, String) - Method in class org.apache.iceberg.spark.Spark3Util.DescribeSchemaVisitor

schemaForTable(SparkSession, String) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Returns a Schema for the given table with fresh field ids.

self() - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction

self() - Method in class org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

self() - Method in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

self() - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

self() - Method in class org.apache.iceberg.spark.actions.RewriteDataFilesSparkAction

self() - Method in class org.apache.iceberg.spark.actions.RewriteManifestsSparkAction

self() - Method in class org.apache.iceberg.spark.actions.RewritePositionDeleteFilesSparkAction

self() - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

sequenceNumber() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

SerializableMetadataTableWithSize(BaseMetadataTable) - Constructor for class org.apache.iceberg.spark.source.SerializableTableWithSize.SerializableMetadataTableWithSize

SerializableTableWithSize - Class in org.apache.iceberg.spark.source

This class provides a serializable table with a known size estimate.

SerializableTableWithSize(Table) - Constructor for class org.apache.iceberg.spark.source.SerializableTableWithSize

SerializableTableWithSize.SerializableMetadataTableWithSize - Class in org.apache.iceberg.spark.source

set(int, T) - Method in class org.apache.iceberg.spark.SparkStructLike

SetAccumulator<T> - Class in org.apache.iceberg.spark.actions

SetAccumulator() - Constructor for class org.apache.iceberg.spark.actions.SetAccumulator

setAddedSnapshotId(Long) - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

setAuthority(String) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

setBatchContext(long) - Method in class org.apache.iceberg.spark.data.SparkOrcReader

setContent(Integer) - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

setDelegateCatalog(CatalogPlugin) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

setDeleteFilter(DeleteFilter<InternalRow>) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnarBatchReader

setJobGroupInfo(SparkContext, JobGroupInfo) - Static method in class org.apache.iceberg.spark.JobGroupUtils

setLength(Long) - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

setOption(String, String, CaseInsensitiveStringMap) - Static method in class org.apache.iceberg.spark.Spark3Util

setPartitionSpecId(Integer) - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

setPath(String) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

setPath(String) - Method in class org.apache.iceberg.spark.actions.FileInfo

setPath(String) - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

setRowGroupInfo(PageReadStore, Map<ColumnPath, ColumnChunkMetaData>, long) - Method in class org.apache.iceberg.spark.data.vectorized.ColumnarBatchReader

setRowPositionSupplier(Supplier<Long>) - Method in class org.apache.iceberg.spark.data.SparkAvroReader

setSchema(Schema) - Method in class org.apache.iceberg.spark.data.SparkAvroReader

setSchema(Schema) - Method in class org.apache.iceberg.spark.data.SparkAvroWriter

setScheme(String) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

setType(String) - Method in class org.apache.iceberg.spark.actions.FileInfo

setUriAsString(String) - Method in class org.apache.iceberg.spark.actions.DeleteOrphanFilesSparkAction.FileURI

shortName() - Method in class org.apache.iceberg.spark.source.IcebergSource

size() - Method in class org.apache.iceberg.spark.SparkStructLike

size() - Method in class org.apache.iceberg.spark.SparkTableCache

SNAPSHOT_ID - Static variable in class org.apache.iceberg.spark.SparkReadOptions

SNAPSHOT_PROPERTY_PREFIX - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

snapshotId() - Method in class org.apache.iceberg.spark.actions.ManifestFileBean

snapshotId() - Method in class org.apache.iceberg.spark.source.SparkTable

snapshotId() - Method in class org.apache.iceberg.spark.SparkReadConf

snapshotTable(String) - Method in class org.apache.iceberg.spark.actions.SparkActions

SnapshotTableSparkAction - Class in org.apache.iceberg.spark.actions

Creates a new Iceberg table based on a source Spark table.

sort(SortOrder) - Method in class org.apache.iceberg.spark.actions.RewriteDataFilesSparkAction

sort() - Method in class org.apache.iceberg.spark.actions.RewriteDataFilesSparkAction

sortOrderId() - Method in class org.apache.iceberg.spark.SparkDataFile

Spark3Util - Class in org.apache.iceberg.spark

Spark3Util.CatalogAndIdentifier - Class in org.apache.iceberg.spark

This mimics a class inside of Spark which is private inside of LookupCatalog.

Spark3Util.DescribeSchemaVisitor - Class in org.apache.iceberg.spark

SPARK_MERGE_SCHEMA - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

SparkActions - Class in org.apache.iceberg.spark.actions

An implementation of ActionsProvider for Spark.

SparkAggregates - Class in org.apache.iceberg.spark

SparkAvroReader - Class in org.apache.iceberg.spark.data

SparkAvroReader(Schema, Schema) - Constructor for class org.apache.iceberg.spark.data.SparkAvroReader

SparkAvroReader(Schema, Schema, Map<Integer, ?>) - Constructor for class org.apache.iceberg.spark.data.SparkAvroReader

SparkAvroWriter - Class in org.apache.iceberg.spark.data

SparkAvroWriter(StructType) - Constructor for class org.apache.iceberg.spark.data.SparkAvroWriter

SparkCachedTableCatalog - Class in org.apache.iceberg.spark

An internal table catalog that is capable of loading tables from a cache.

SparkCachedTableCatalog() - Constructor for class org.apache.iceberg.spark.SparkCachedTableCatalog

SparkCatalog - Class in org.apache.iceberg.spark

A Spark TableCatalog implementation that wraps an Iceberg Catalog.

SparkCatalog() - Constructor for class org.apache.iceberg.spark.SparkCatalog

SparkChangelogTable - Class in org.apache.iceberg.spark.source

SparkChangelogTable(Table, boolean) - Constructor for class org.apache.iceberg.spark.source.SparkChangelogTable

SparkDataFile - Class in org.apache.iceberg.spark

SparkDataFile(Types.StructType, StructType) - Constructor for class org.apache.iceberg.spark.SparkDataFile

SparkDataFile(Types.StructType, Types.StructType, StructType) - Constructor for class org.apache.iceberg.spark.SparkDataFile

SparkDistributionAndOrderingUtil - Class in org.apache.iceberg.spark

SparkExceptionUtil - Class in org.apache.iceberg.spark

SparkFilters - Class in org.apache.iceberg.spark

SparkFunctions - Class in org.apache.iceberg.spark.functions

SparkMetadataColumn - Class in org.apache.iceberg.spark.source

SparkMetadataColumn(String, DataType, boolean) - Constructor for class org.apache.iceberg.spark.source.SparkMetadataColumn

SparkMicroBatchStream - Class in org.apache.iceberg.spark.source

SparkOrcReader - Class in org.apache.iceberg.spark.data

Converts the OrcIterator, which returns ORC's VectorizedRowBatch to a set of Spark's UnsafeRows.

SparkOrcReader(Schema, TypeDescription) - Constructor for class org.apache.iceberg.spark.data.SparkOrcReader

SparkOrcReader(Schema, TypeDescription, Map<Integer, ?>) - Constructor for class org.apache.iceberg.spark.data.SparkOrcReader

SparkOrcValueReaders - Class in org.apache.iceberg.spark.data

SparkOrcWriter - Class in org.apache.iceberg.spark.data

This class acts as an adaptor from an OrcFileAppender to a FileAppender<InternalRow>.

SparkOrcWriter(Schema, TypeDescription) - Constructor for class org.apache.iceberg.spark.data.SparkOrcWriter

SparkParquetReaders - Class in org.apache.iceberg.spark.data

SparkParquetWriters - Class in org.apache.iceberg.spark.data

SparkPartition(Map<String, String>, String, String) - Constructor for class org.apache.iceberg.spark.SparkTableUtil.SparkPartition

SparkPartitionedFanoutWriter - Class in org.apache.iceberg.spark.source

SparkPartitionedFanoutWriter(PartitionSpec, FileFormat, FileAppenderFactory<InternalRow>, OutputFileFactory, FileIO, long, Schema, StructType) - Constructor for class org.apache.iceberg.spark.source.SparkPartitionedFanoutWriter

SparkPartitionedWriter - Class in org.apache.iceberg.spark.source

SparkPartitionedWriter(PartitionSpec, FileFormat, FileAppenderFactory<InternalRow>, OutputFileFactory, FileIO, long, Schema, StructType) - Constructor for class org.apache.iceberg.spark.source.SparkPartitionedWriter

SparkPositionDeletesRewrite - Class in org.apache.iceberg.spark.source

Write class for rewriting position delete files from Spark.

SparkPositionDeletesRewrite.DeleteTaskCommit - Class in org.apache.iceberg.spark.source

SparkPositionDeletesRewriteBuilder - Class in org.apache.iceberg.spark.source

Builder class for rewrites of position delete files from Spark.

SparkProcedures - Class in org.apache.iceberg.spark.procedures

SparkProcedures.ProcedureBuilder - Interface in org.apache.iceberg.spark.procedures

SparkReadConf - Class in org.apache.iceberg.spark

A class for common Iceberg configs for Spark reads.

SparkReadConf(SparkSession, Table, Map<String, String>) - Constructor for class org.apache.iceberg.spark.SparkReadConf

SparkReadConf(SparkSession, Table, String, Map<String, String>) - Constructor for class org.apache.iceberg.spark.SparkReadConf

SparkReadOptions - Class in org.apache.iceberg.spark

Spark DF read options

SparkScanBuilder - Class in org.apache.iceberg.spark.source

SparkSchemaUtil - Class in org.apache.iceberg.spark

Helper methods for working with Spark/Hive metadata.

SparkSessionCatalog<T extends org.apache.spark.sql.connector.catalog.TableCatalog & org.apache.spark.sql.connector.catalog.FunctionCatalog & org.apache.spark.sql.connector.catalog.SupportsNamespaces> - Class in org.apache.iceberg.spark

A Spark catalog that can also load non-Iceberg tables.

SparkSessionCatalog() - Constructor for class org.apache.iceberg.spark.SparkSessionCatalog

SparkSQLProperties - Class in org.apache.iceberg.spark

SparkStructLike - Class in org.apache.iceberg.spark

SparkStructLike(Types.StructType) - Constructor for class org.apache.iceberg.spark.SparkStructLike

SparkTable - Class in org.apache.iceberg.spark.source

SparkTable(Table, boolean) - Constructor for class org.apache.iceberg.spark.source.SparkTable

SparkTable(Table, String, boolean) - Constructor for class org.apache.iceberg.spark.source.SparkTable

SparkTable(Table, Long, boolean) - Constructor for class org.apache.iceberg.spark.source.SparkTable

SparkTableCache - Class in org.apache.iceberg.spark

SparkTableCache() - Constructor for class org.apache.iceberg.spark.SparkTableCache

SparkTableUtil - Class in org.apache.iceberg.spark

Java version of the original SparkTableUtil.scala https://github.com/apache/iceberg/blob/apache-iceberg-0.8.0-incubating/spark/src/main/scala/org/apache/iceberg/spark/SparkTableUtil.scala

SparkTableUtil.SparkPartition - Class in org.apache.iceberg.spark

Class representing a table partition.

SparkUtil - Class in org.apache.iceberg.spark

SparkV2Filters - Class in org.apache.iceberg.spark

SparkValueConverter - Class in org.apache.iceberg.spark

A utility class that converts Spark values to Iceberg's internal representation.

SparkValueReaders - Class in org.apache.iceberg.spark.data

SparkValueWriters - Class in org.apache.iceberg.spark.data

SparkWriteConf - Class in org.apache.iceberg.spark

A class for common Iceberg configs for Spark writes.

SparkWriteConf(SparkSession, Table, Map<String, String>) - Constructor for class org.apache.iceberg.spark.SparkWriteConf

SparkWriteConf(SparkSession, Table, String, Map<String, String>) - Constructor for class org.apache.iceberg.spark.SparkWriteConf

SparkWriteOptions - Class in org.apache.iceberg.spark

Spark DF write options

specForTable(SparkSession, String) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

Returns a PartitionSpec for the given table.

specId(int) - Method in class org.apache.iceberg.spark.actions.RewriteManifestsSparkAction

specId() - Method in class org.apache.iceberg.spark.SparkDataFile

SPLIT_SIZE - Static variable in class org.apache.iceberg.spark.SparkReadOptions

splitLookback() - Method in class org.apache.iceberg.spark.SparkReadConf

splitLookbackOption() - Method in class org.apache.iceberg.spark.SparkReadConf

splitOffsets() - Method in class org.apache.iceberg.spark.SparkDataFile

splitOpenFileCost() - Method in class org.apache.iceberg.spark.SparkReadConf

splitOpenFileCostOption() - Method in class org.apache.iceberg.spark.SparkReadConf

splitSize() - Method in class org.apache.iceberg.spark.SparkReadConf

splitSizeOption() - Method in class org.apache.iceberg.spark.SparkReadConf

stageCreate(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkCatalog

stageCreate(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

stageCreateOrReplace(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkCatalog

stageCreateOrReplace(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

StagedSparkTable - Class in org.apache.iceberg.spark.source

StagedSparkTable(Transaction) - Constructor for class org.apache.iceberg.spark.source.StagedSparkTable

stageReplace(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkCatalog

stageReplace(Identifier, StructType, Transform[], Map<String, String>) - Method in class org.apache.iceberg.spark.SparkSessionCatalog

stageTasks(Table, String, List<T>) - Method in class org.apache.iceberg.spark.ScanTaskSetManager

stagingLocation(String) - Method in class org.apache.iceberg.spark.actions.RewriteManifestsSparkAction

START_SNAPSHOT_ID - Static variable in class org.apache.iceberg.spark.SparkReadOptions

START_TIMESTAMP - Static variable in class org.apache.iceberg.spark.SparkReadOptions

startSnapshotId() - Method in class org.apache.iceberg.spark.SparkReadConf

startTimestamp() - Method in class org.apache.iceberg.spark.SparkReadConf

stop() - Method in class org.apache.iceberg.spark.source.SparkMicroBatchStream

STREAM_FROM_TIMESTAMP - Static variable in class org.apache.iceberg.spark.SparkReadOptions

STREAM_RESULTS - Static variable in class org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

STREAM_RESULTS - Static variable in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

STREAM_RESULTS_DEFAULT - Static variable in class org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

STREAM_RESULTS_DEFAULT - Static variable in class org.apache.iceberg.spark.actions.ExpireSnapshotsSparkAction

streamFromTimestamp() - Method in class org.apache.iceberg.spark.SparkReadConf

STREAMING_MAX_FILES_PER_MICRO_BATCH - Static variable in class org.apache.iceberg.spark.SparkReadOptions

STREAMING_MAX_ROWS_PER_MICRO_BATCH - Static variable in class org.apache.iceberg.spark.SparkReadOptions

STREAMING_SKIP_DELETE_SNAPSHOTS - Static variable in class org.apache.iceberg.spark.SparkReadOptions

STREAMING_SKIP_DELETE_SNAPSHOTS_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkReadOptions

STREAMING_SKIP_OVERWRITE_SNAPSHOTS - Static variable in class org.apache.iceberg.spark.SparkReadOptions

STREAMING_SKIP_OVERWRITE_SNAPSHOTS_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkReadOptions

streamingSkipDeleteSnapshots() - Method in class org.apache.iceberg.spark.SparkReadConf

streamingSkipOverwriteSnapshots() - Method in class org.apache.iceberg.spark.SparkReadConf

struct(StructType, GroupType, List<T>) - Method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

struct(Types.StructType, Iterable<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithoutReordering

struct(Types.StructType, Iterable<Type>) - Method in class org.apache.iceberg.spark.PruneColumnsWithReordering

struct(Types.StructType, List<String>) - Method in class org.apache.iceberg.spark.Spark3Util.DescribeSchemaVisitor

SupportsDelta - Interface in org.apache.spark.sql.connector.iceberg.write

A mix-in interface for RowLevelOperation.

supportsExternalMetadata() - Method in class org.apache.iceberg.spark.source.IcebergSource

T

table() - Method in class org.apache.iceberg.spark.source.SparkTable

TABLE_NAME - Static variable in class org.apache.iceberg.spark.source.SparkChangelogTable

tableLocation(String) - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

tableProperties(Map<String, String>) - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

tableProperties(Map<String, String>) - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

tableProperty(String, String) - Method in class org.apache.iceberg.spark.actions.MigrateTableSparkAction

tableProperty(String, String) - Method in class org.apache.iceberg.spark.actions.SnapshotTableSparkAction

tag() - Method in class org.apache.iceberg.spark.SparkReadConf

TAG - Static variable in class org.apache.iceberg.spark.SparkReadOptions

TARGET_DELETE_FILE_SIZE_BYTES - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

TARGET_FILE_SIZE_BYTES - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

targetDataFileSize() - Method in class org.apache.iceberg.spark.SparkWriteConf

targetDeleteFileSize() - Method in class org.apache.iceberg.spark.SparkWriteConf

TaskNumDeletes - Class in org.apache.iceberg.spark.source.metrics

TaskNumDeletes(long) - Constructor for class org.apache.iceberg.spark.source.metrics.TaskNumDeletes

TaskNumSplits - Class in org.apache.iceberg.spark.source.metrics

TaskNumSplits(long) - Constructor for class org.apache.iceberg.spark.source.metrics.TaskNumSplits

term() - Method in class org.apache.iceberg.spark.ExtendedParser.RawOrderField

TIMESTAMP_AS_OF - Static variable in class org.apache.iceberg.spark.SparkReadOptions

TIMESTAMP_WITHOUT_TIMEZONE_ERROR - Static variable in class org.apache.iceberg.spark.SparkUtil

TimestampToDaysFunction() - Constructor for class org.apache.iceberg.spark.functions.DaysFunction.TimestampToDaysFunction

TimestampToHoursFunction() - Constructor for class org.apache.iceberg.spark.functions.HoursFunction.TimestampToHoursFunction

TimestampToMonthsFunction() - Constructor for class org.apache.iceberg.spark.functions.MonthsFunction.TimestampToMonthsFunction

TimestampToYearsFunction() - Constructor for class org.apache.iceberg.spark.functions.YearsFunction.TimestampToYearsFunction

timestampTzs() - Static method in class org.apache.iceberg.spark.data.SparkOrcValueReaders

toBatch() - Method in class org.apache.iceberg.spark.source.SparkPositionDeletesRewrite

toBatch() - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaWrite

toColumnName(NamedReference) - Static method in class org.apache.iceberg.spark.SparkUtil

toIcebergTable(Table) - Static method in class org.apache.iceberg.spark.Spark3Util

toIcebergTerm(Expression) - Static method in class org.apache.iceberg.spark.Spark3Util

toNamedReference(String) - Static method in class org.apache.iceberg.spark.Spark3Util

toPartitionSpec(Schema, Transform[]) - Static method in class org.apache.iceberg.spark.Spark3Util

Converts Spark transforms into a PartitionSpec.

toString() - Method in class org.apache.iceberg.spark.source.SparkTable

toString() - Method in class org.apache.iceberg.spark.SparkTableUtil.SparkPartition

toTransforms(Schema, List<PartitionField>) - Static method in class org.apache.iceberg.spark.Spark3Util

toTransforms(PartitionSpec) - Static method in class org.apache.iceberg.spark.Spark3Util

Converts a PartitionSpec to Spark transforms.

toUncheckedException(Throwable, String, Object...) - Static method in class org.apache.iceberg.spark.SparkExceptionUtil

Converts checked exceptions to unchecked exceptions.

toV1TableIdentifier(Identifier) - Static method in class org.apache.iceberg.spark.Spark3Util

TruncateBase() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBase

TruncateBigInt() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBigInt

TruncateBinary() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateBinary

TruncateDecimal(int, int) - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateDecimal

TruncateFunction - Class in org.apache.iceberg.spark.functions

A Spark function implementation for the Iceberg truncate transform.

TruncateFunction() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction

TruncateFunction.TruncateBase<T> - Class in org.apache.iceberg.spark.functions

TruncateFunction.TruncateBigInt - Class in org.apache.iceberg.spark.functions

TruncateFunction.TruncateBinary - Class in org.apache.iceberg.spark.functions

TruncateFunction.TruncateDecimal - Class in org.apache.iceberg.spark.functions

TruncateFunction.TruncateInt - Class in org.apache.iceberg.spark.functions

TruncateFunction.TruncateSmallInt - Class in org.apache.iceberg.spark.functions

TruncateFunction.TruncateString - Class in org.apache.iceberg.spark.functions

TruncateFunction.TruncateTinyInt - Class in org.apache.iceberg.spark.functions

TruncateInt() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateInt

TruncateSmallInt() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateSmallInt

TruncateString() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateString

TruncateTinyInt() - Constructor for class org.apache.iceberg.spark.functions.TruncateFunction.TruncateTinyInt

U

update(T, T, T) - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaWriter

Passes information for a row that must be updated together with the updated row.

UPDATE_AFTER - Static variable in class org.apache.iceberg.spark.ChangelogIterator

UPDATE_BEFORE - Static variable in class org.apache.iceberg.spark.ChangelogIterator

updateDistributionMode() - Method in class org.apache.iceberg.spark.SparkWriteConf

upperBounds() - Method in class org.apache.iceberg.spark.SparkDataFile

USE_CACHING - Static variable in class org.apache.iceberg.spark.actions.RewriteManifestsSparkAction

USE_CACHING_DEFAULT - Static variable in class org.apache.iceberg.spark.actions.RewriteManifestsSparkAction

USE_TABLE_DISTRIBUTION_AND_ORDERING - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

USE_TABLE_DISTRIBUTION_AND_ORDERING_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

USE_TIMESTAMP_WITHOUT_TIME_ZONE_IN_NEW_TABLES - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

USE_TIMESTAMP_WITHOUT_TIME_ZONE_IN_NEW_TABLES_DEFAULT - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

useTableDistributionAndOrdering() - Method in class org.apache.iceberg.spark.SparkWriteConf

useTimestampWithoutZoneInNewTables(RuntimeConfig) - Static method in class org.apache.iceberg.spark.SparkUtil

Checks whether timestamp types for new tables should be stored with timezone info.

utf8String() - Static method in class org.apache.iceberg.spark.data.SparkOrcValueReaders

uuids() - Static method in class org.apache.iceberg.spark.data.SparkOrcValueReaders

V

VALIDATE_FROM_SNAPSHOT_ID - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

validateFromSnapshotId() - Method in class org.apache.iceberg.spark.SparkWriteConf

validateMetadataColumnReferences(Schema, Schema) - Static method in class org.apache.iceberg.spark.SparkSchemaUtil

validatePartitionTransforms(PartitionSpec) - Static method in class org.apache.iceberg.spark.SparkUtil

Check whether the partition transforms in a spec can be used to write data.

value() - Method in class org.apache.iceberg.spark.actions.SetAccumulator

value() - Method in class org.apache.iceberg.spark.source.metrics.TaskNumDeletes

value() - Method in class org.apache.iceberg.spark.source.metrics.TaskNumSplits

valueCounts() - Method in class org.apache.iceberg.spark.SparkDataFile

vectorAccessor() - Method in class org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

VECTORIZATION_BATCH_SIZE - Static variable in class org.apache.iceberg.spark.SparkReadOptions

VECTORIZATION_ENABLED - Static variable in class org.apache.iceberg.spark.SparkReadOptions

VECTORIZATION_ENABLED - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

VectorizedSparkOrcReaders - Class in org.apache.iceberg.spark.data.vectorized

VectorizedSparkParquetReaders - Class in org.apache.iceberg.spark.data.vectorized

VERSION_AS_OF - Static variable in class org.apache.iceberg.spark.SparkReadOptions

visit(DataType, Type, ParquetWithSparkSchemaVisitor<T>) - Static method in class org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor

W

WAP_BRANCH - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

WAP_ID - Static variable in class org.apache.iceberg.spark.SparkSQLProperties

wapEnabled(Table) - Static method in class org.apache.iceberg.spark.SparkTableUtil

wapEnabled() - Method in class org.apache.iceberg.spark.SparkWriteConf

wapId() - Method in class org.apache.iceberg.spark.SparkWriteConf

withCommitProperties(Map<String, String>, Callable<R>, Class<E>) - Static method in class org.apache.iceberg.spark.CommitMetadata

running the code wrapped as a caller, and any snapshot committed within the callable object will be attached with the metadata defined in properties

withTableCatalog(TableCatalog) - Method in interface org.apache.iceberg.spark.procedures.SparkProcedures.ProcedureBuilder

wrap(Row) - Method in class org.apache.iceberg.spark.SparkDataFile

wrap(Row) - Method in class org.apache.iceberg.spark.SparkStructLike

write(InternalRow, Encoder) - Method in class org.apache.iceberg.spark.data.SparkAvroWriter

write(InternalRow, VectorizedRowBatch) - Method in class org.apache.iceberg.spark.data.SparkOrcWriter

write(T) - Method in interface org.apache.spark.sql.connector.iceberg.write.DeltaWriter

WRITE_FORMAT - Static variable in class org.apache.iceberg.spark.SparkWriteOptions

writers() - Method in class org.apache.iceberg.spark.data.SparkOrcWriter

Y

YearsFunction - Class in org.apache.iceberg.spark.functions

A Spark function implementation for the Iceberg year transform.

YearsFunction() - Constructor for class org.apache.iceberg.spark.functions.YearsFunction

YearsFunction.DateToYearsFunction - Class in org.apache.iceberg.spark.functions

YearsFunction.TimestampToYearsFunction - Class in org.apache.iceberg.spark.functions

Z

zOrder(String...) - Method in class org.apache.iceberg.spark.actions.RewriteDataFilesSparkAction

A B C D E F G H I J K L M N O P Q R S T U V W Y Z