Package io.trino.tests.product.iceberg

Class TestIcebergSparkCompatibility

java.lang.Object

io.trino.tempto.ProductTest

io.trino.tests.product.iceberg.TestIcebergSparkCompatibility

public class TestIcebergSparkCompatibility
extends io.trino.tempto.ProductTest

Tests compatibility between Iceberg connector and Spark Iceberg.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	TestIcebergSparkCompatibility.CreateMode
static class	TestIcebergSparkCompatibility.StorageFormat

Constructor Summary

Constructors

Constructor	Description
TestIcebergSparkCompatibility()

Method Summary

Modifier and Type	Method	Description
static Object[][]	specVersions()
static Object[][]	storageFormats()
Object[][]	storageFormatsAndCompressionCodecs()
static Object[][]	storageFormatsWithSpecVersion()
static Object[][]	tableFormatWithDeleteFormat()
void	testCleaningUpIcebergTableWithRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat, TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)
void	testDeleteAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testHandlingPartitionSchemaEvolutionInPartitionMetadata()
void	testIdBasedFieldMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testInsertReadingFromParquetTableWithNestedRowFieldNotPresentInDataFile()
void	testMigratedDataWithAlteredSchema(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testMigratedDataWithPartialNameMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testMissingMetrics()
void	testOptimizeOnV2IcebergTable()
void	testPartialStats()
void	testReadAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testSparkCreatesTrinoDrops(int specVersion)
void	testSparkReadingCompositeTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadingNestedTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadingTrinoCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat, String compressionCodec)
void	testSparkReadingTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat, TestIcebergSparkCompatibility.CreateMode createMode)
Object[][]	testSparkReadingTrinoDataDataProvider()
void	testSparkReadsTrinoPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoTableAfterCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoTableAfterOptimizeAndCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testSparkReadTrinoUuid(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testStatsAfterAddingPartitionField()
void	testStringPartitioningWithSpecialCharactersCtasInTrino()
void	testStringPartitioningWithSpecialCharactersInsertInSpark()
void	testStringPartitioningWithSpecialCharactersInsertInTrino()
void	testTrinoCreatesSparkDrops()
void	testTrinoReadingCompositeSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadingMigratedNestedData(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testTrinoReadingNestedSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadingSparkCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat, String compressionCodec)
void	testTrinoReadingSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadingSparkIcebergTablePropertiesData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadsSparkPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadsSparkRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat, TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)
void	testTrinoReadsSparkRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat, TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)
void	testTrinoReadsTrinoTableWithSparkDeletesAfterOptimizeAndCleanUp(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoShowingSparkCreatedTables(int specVersion)
void	testTrinoSparkConcurrentInsert()
void	testTrinoWithUnsupportedFileFormat(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testTrinoWritingDataWithObjectStorageLocationProvider(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoWritingDataWithWriterDataPathSet(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testUpdateAfterSchemaEvolution()
void	testUpdateOnPartitionColumn()
static Object[][]	unsupportedStorageFormats()
void	verifyCompressionCodecsDataProvider()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TestIcebergSparkCompatibility

public TestIcebergSparkCompatibility()

Method Detail

testTrinoWithUnsupportedFileFormat

public void testTrinoWithUnsupportedFileFormat(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testTrinoReadingSparkData

public void testTrinoReadingSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                      int specVersion)

testSparkReadingTrinoData

public void testSparkReadingTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                      TestIcebergSparkCompatibility.CreateMode createMode)

testSparkReadingTrinoDataDataProvider

public Object[][] testSparkReadingTrinoDataDataProvider()

testSparkReadTrinoUuid

public void testSparkReadTrinoUuid(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkCreatesTrinoDrops

public void testSparkCreatesTrinoDrops(int specVersion)

testTrinoCreatesSparkDrops

public void testTrinoCreatesSparkDrops()

testSparkReadsTrinoPartitionedTable

public void testSparkReadsTrinoPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testTrinoReadsSparkPartitionedTable

public void testTrinoReadsSparkPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                int specVersion)

testTrinoReadingCompositeSparkData

public void testTrinoReadingCompositeSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                               int specVersion)

testSparkReadingCompositeTrinoData

public void testSparkReadingCompositeTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testTrinoReadingSparkIcebergTablePropertiesData

public void testTrinoReadingSparkIcebergTablePropertiesData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                            int specVersion)

testTrinoReadingNestedSparkData

public void testTrinoReadingNestedSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                            int specVersion)

testSparkReadingNestedTrinoData

public void testSparkReadingNestedTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testIdBasedFieldMapping

public void testIdBasedFieldMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                    int specVersion)

testReadAfterPartitionEvolution

public void testReadAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                            int specVersion)

testTrinoShowingSparkCreatedTables

public void testTrinoShowingSparkCreatedTables(int specVersion)

testTrinoWritingDataWithObjectStorageLocationProvider

public void testTrinoWritingDataWithObjectStorageLocationProvider(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                                  int specVersion)

testTrinoWritingDataWithWriterDataPathSet

public void testTrinoWritingDataWithWriterDataPathSet(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                      int specVersion)

testStringPartitioningWithSpecialCharactersCtasInTrino

public void testStringPartitioningWithSpecialCharactersCtasInTrino()

testStringPartitioningWithSpecialCharactersInsertInTrino

public void testStringPartitioningWithSpecialCharactersInsertInTrino()

testStringPartitioningWithSpecialCharactersInsertInSpark

public void testStringPartitioningWithSpecialCharactersInsertInSpark()

testInsertReadingFromParquetTableWithNestedRowFieldNotPresentInDataFile

public void testInsertReadingFromParquetTableWithNestedRowFieldNotPresentInDataFile()

testTrinoSparkConcurrentInsert

public void testTrinoSparkConcurrentInsert()
                                    throws Exception

Throws:

Exception

See Also:

TestIcebergInsert.testIcebergConcurrentInsert()

testTrinoReadingSparkCompressedData

public void testTrinoReadingSparkCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                String compressionCodec)

testSparkReadingTrinoCompressedData

public void testSparkReadingTrinoCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                String compressionCodec)

verifyCompressionCodecsDataProvider

public void verifyCompressionCodecsDataProvider()

storageFormatsAndCompressionCodecs

public Object[][] storageFormatsAndCompressionCodecs()

testTrinoReadingMigratedNestedData

public void testTrinoReadingMigratedNestedData(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testMigratedDataWithAlteredSchema

public void testMigratedDataWithAlteredSchema(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testMigratedDataWithPartialNameMapping

public void testMigratedDataWithPartialNameMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testPartialStats

public void testPartialStats()

testStatsAfterAddingPartitionField

public void testStatsAfterAddingPartitionField()

testTrinoReadsSparkRowLevelDeletes

public void testTrinoReadsSparkRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat,
                                               TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)

testTrinoReadsSparkRowLevelDeletesWithRowTypes

public void testTrinoReadsSparkRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat,
                                                           TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)

testSparkReadsTrinoRowLevelDeletes

public void testSparkReadsTrinoRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkReadsTrinoRowLevelDeletesWithRowTypes

public void testSparkReadsTrinoRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testDeleteAfterPartitionEvolution

public void testDeleteAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testMissingMetrics

public void testMissingMetrics()

testOptimizeOnV2IcebergTable

public void testOptimizeOnV2IcebergTable()

specVersions

public static Object[][] specVersions()

storageFormats

public static Object[][] storageFormats()

tableFormatWithDeleteFormat

public static Object[][] tableFormatWithDeleteFormat()

storageFormatsWithSpecVersion

public static Object[][] storageFormatsWithSpecVersion()

unsupportedStorageFormats

public static Object[][] unsupportedStorageFormats()

testSparkReadsTrinoTableAfterCleaningUp

public void testSparkReadsTrinoTableAfterCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkReadsTrinoTableAfterOptimizeAndCleaningUp

public void testSparkReadsTrinoTableAfterOptimizeAndCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                               int specVersion)

testTrinoReadsTrinoTableWithSparkDeletesAfterOptimizeAndCleanUp

public void testTrinoReadsTrinoTableWithSparkDeletesAfterOptimizeAndCleanUp(TestIcebergSparkCompatibility.StorageFormat storageFormat,
                                                                            int specVersion)

testCleaningUpIcebergTableWithRowLevelDeletes

public void testCleaningUpIcebergTableWithRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat,
                                                          TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)

testUpdateAfterSchemaEvolution

public void testUpdateAfterSchemaEvolution()

testUpdateOnPartitionColumn

public void testUpdateOnPartitionColumn()

testHandlingPartitionSchemaEvolutionInPartitionMetadata

public void testHandlingPartitionSchemaEvolutionInPartitionMetadata()