org.apache.iceberg.spark.actions

Class DeleteReachableFilesSparkAction

java.lang.Object

org.apache.iceberg.spark.actions.DeleteReachableFilesSparkAction

All Implemented Interfaces:

org.apache.iceberg.actions.Action<org.apache.iceberg.actions.DeleteReachableFiles,org.apache.iceberg.actions.DeleteReachableFiles.Result>, org.apache.iceberg.actions.DeleteReachableFiles

public class DeleteReachableFilesSparkAction
extends java.lang.Object
implements org.apache.iceberg.actions.DeleteReachableFiles

An implementation of DeleteReachableFiles that uses metadata tables in Spark to determine which files should be deleted.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.iceberg.actions.DeleteReachableFiles

org.apache.iceberg.actions.DeleteReachableFiles.Result

Field Summary

Fields

Modifier and Type	Field and Description
protected static org.apache.iceberg.relocated.com.google.common.base.Joiner	COMMA_JOINER
protected static org.apache.iceberg.relocated.com.google.common.base.Splitter	COMMA_SPLITTER
protected static java.lang.String	FILE_PATH
protected static java.lang.String	LAST_MODIFIED
protected static java.lang.String	MANIFEST
protected static java.lang.String	MANIFEST_LIST
protected static java.lang.String	OTHERS
protected static java.lang.String	STATISTICS_FILES
static java.lang.String	STREAM_RESULTS
static boolean	STREAM_RESULTS_DEFAULT

Method Summary

Modifier and Type	Method and Description
protected org.apache.spark.sql.Dataset<FileInfo>	allReachableOtherMetadataFileDS(org.apache.iceberg.Table table)
protected org.apache.spark.sql.Dataset<FileInfo>	contentFileDS(org.apache.iceberg.Table table)
protected org.apache.spark.sql.Dataset<FileInfo>	contentFileDS(org.apache.iceberg.Table table, java.util.Set<java.lang.Long> snapshotIds)
protected org.apache.iceberg.spark.actions.BaseSparkAction.DeleteSummary	deleteFiles(java.util.concurrent.ExecutorService executorService, java.util.function.Consumer<java.lang.String> deleteFunc, java.util.Iterator<FileInfo> files) Deletes files and keeps track of how many files were removed for each file type.
protected org.apache.iceberg.spark.actions.BaseSparkAction.DeleteSummary	deleteFiles(org.apache.iceberg.io.SupportsBulkOperations io, java.util.Iterator<FileInfo> files)
DeleteReachableFilesSparkAction	deleteWith(java.util.function.Consumer<java.lang.String> newDeleteFunc)
org.apache.iceberg.actions.DeleteReachableFiles.Result	execute()
DeleteReachableFilesSparkAction	executeDeleteWith(java.util.concurrent.ExecutorService executorService)
DeleteReachableFilesSparkAction	io(org.apache.iceberg.io.FileIO fileIO)
protected org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	loadMetadataTable(org.apache.iceberg.Table table, org.apache.iceberg.MetadataTableType type)
protected org.apache.spark.sql.Dataset<FileInfo>	manifestDS(org.apache.iceberg.Table table)
protected org.apache.spark.sql.Dataset<FileInfo>	manifestDS(org.apache.iceberg.Table table, java.util.Set<java.lang.Long> snapshotIds)
protected org.apache.spark.sql.Dataset<FileInfo>	manifestListDS(org.apache.iceberg.Table table)
protected org.apache.spark.sql.Dataset<FileInfo>	manifestListDS(org.apache.iceberg.Table table, java.util.Set<java.lang.Long> snapshotIds)
protected JobGroupInfo	newJobGroupInfo(java.lang.String groupId, java.lang.String desc)
protected org.apache.iceberg.Table	newStaticTable(org.apache.iceberg.TableMetadata metadata, org.apache.iceberg.io.FileIO io)
ThisT	option(java.lang.String name, java.lang.String value)
protected java.util.Map<java.lang.String,java.lang.String>	options()
ThisT	options(java.util.Map<java.lang.String,java.lang.String> newOptions)
protected org.apache.spark.sql.Dataset<FileInfo>	otherMetadataFileDS(org.apache.iceberg.Table table)
protected DeleteReachableFilesSparkAction	self()
protected org.apache.spark.sql.SparkSession	spark()
protected org.apache.spark.api.java.JavaSparkContext	sparkContext()
protected org.apache.spark.sql.Dataset<FileInfo>	statisticsFileDS(org.apache.iceberg.Table table, java.util.Set<java.lang.Long> snapshotIds)
protected <T> T	withJobGroupInfo(JobGroupInfo info, java.util.function.Supplier<T> supplier)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.iceberg.actions.Action

option, options

Field Detail

STREAM_RESULTS

public static final java.lang.String STREAM_RESULTS

See Also:

Constant Field Values

STREAM_RESULTS_DEFAULT

public static final boolean STREAM_RESULTS_DEFAULT

See Also:

Constant Field Values

MANIFEST

protected static final java.lang.String MANIFEST

See Also:

Constant Field Values

MANIFEST_LIST

protected static final java.lang.String MANIFEST_LIST

See Also:

Constant Field Values

STATISTICS_FILES

protected static final java.lang.String STATISTICS_FILES

See Also:

Constant Field Values

OTHERS

protected static final java.lang.String OTHERS

See Also:

Constant Field Values

FILE_PATH

protected static final java.lang.String FILE_PATH

See Also:

Constant Field Values

LAST_MODIFIED

protected static final java.lang.String LAST_MODIFIED

See Also:

Constant Field Values

COMMA_SPLITTER

protected static final org.apache.iceberg.relocated.com.google.common.base.Splitter COMMA_SPLITTER

COMMA_JOINER

protected static final org.apache.iceberg.relocated.com.google.common.base.Joiner COMMA_JOINER

Method Detail

self

protected DeleteReachableFilesSparkAction self()

io

public DeleteReachableFilesSparkAction io(org.apache.iceberg.io.FileIO fileIO)

Specified by:

io in interface org.apache.iceberg.actions.DeleteReachableFiles

deleteWith

public DeleteReachableFilesSparkAction deleteWith(java.util.function.Consumer<java.lang.String> newDeleteFunc)

Specified by:

deleteWith in interface org.apache.iceberg.actions.DeleteReachableFiles

executeDeleteWith

public DeleteReachableFilesSparkAction executeDeleteWith(java.util.concurrent.ExecutorService executorService)

Specified by:

executeDeleteWith in interface org.apache.iceberg.actions.DeleteReachableFiles

execute

public org.apache.iceberg.actions.DeleteReachableFiles.Result execute()

Specified by:

execute in interface org.apache.iceberg.actions.Action<org.apache.iceberg.actions.DeleteReachableFiles,org.apache.iceberg.actions.DeleteReachableFiles.Result>

spark

protected org.apache.spark.sql.SparkSession spark()

sparkContext

protected org.apache.spark.api.java.JavaSparkContext sparkContext()

option

public ThisT option(java.lang.String name,
                    java.lang.String value)

options

public ThisT options(java.util.Map<java.lang.String,java.lang.String> newOptions)

options

protected java.util.Map<java.lang.String,java.lang.String> options()

withJobGroupInfo

protected <T> T withJobGroupInfo(JobGroupInfo info,
                                 java.util.function.Supplier<T> supplier)

newJobGroupInfo

protected JobGroupInfo newJobGroupInfo(java.lang.String groupId,
                                       java.lang.String desc)

newStaticTable

protected org.apache.iceberg.Table newStaticTable(org.apache.iceberg.TableMetadata metadata,
                                                  org.apache.iceberg.io.FileIO io)

contentFileDS

protected org.apache.spark.sql.Dataset<FileInfo> contentFileDS(org.apache.iceberg.Table table)

contentFileDS

protected org.apache.spark.sql.Dataset<FileInfo> contentFileDS(org.apache.iceberg.Table table,
                                                               java.util.Set<java.lang.Long> snapshotIds)

manifestDS

protected org.apache.spark.sql.Dataset<FileInfo> manifestDS(org.apache.iceberg.Table table)

manifestDS

protected org.apache.spark.sql.Dataset<FileInfo> manifestDS(org.apache.iceberg.Table table,
                                                            java.util.Set<java.lang.Long> snapshotIds)

manifestListDS

protected org.apache.spark.sql.Dataset<FileInfo> manifestListDS(org.apache.iceberg.Table table)

manifestListDS

protected org.apache.spark.sql.Dataset<FileInfo> manifestListDS(org.apache.iceberg.Table table,
                                                                java.util.Set<java.lang.Long> snapshotIds)

statisticsFileDS

protected org.apache.spark.sql.Dataset<FileInfo> statisticsFileDS(org.apache.iceberg.Table table,
                                                                  java.util.Set<java.lang.Long> snapshotIds)

otherMetadataFileDS

protected org.apache.spark.sql.Dataset<FileInfo> otherMetadataFileDS(org.apache.iceberg.Table table)

allReachableOtherMetadataFileDS

protected org.apache.spark.sql.Dataset<FileInfo> allReachableOtherMetadataFileDS(org.apache.iceberg.Table table)

loadMetadataTable

protected org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> loadMetadataTable(org.apache.iceberg.Table table,
                                                                                   org.apache.iceberg.MetadataTableType type)

deleteFiles

protected org.apache.iceberg.spark.actions.BaseSparkAction.DeleteSummary deleteFiles(java.util.concurrent.ExecutorService executorService,
                                                                                     java.util.function.Consumer<java.lang.String> deleteFunc,
                                                                                     java.util.Iterator<FileInfo> files)

Deletes files and keeps track of how many files were removed for each file type.

Parameters:

executorService - an executor service to use for parallel deletes

deleteFunc - a delete func

files - an iterator of Spark rows of the structure (path: String, type: String)

Returns:

stats on which files were deleted

deleteFiles

protected org.apache.iceberg.spark.actions.BaseSparkAction.DeleteSummary deleteFiles(org.apache.iceberg.io.SupportsBulkOperations io,
                                                                                     java.util.Iterator<FileInfo> files)