Package io.trino.tests.product.iceberg

Class TestIcebergSparkCompatibility

java.lang.Object

io.trino.tempto.ProductTest

io.trino.tests.product.iceberg.TestIcebergSparkCompatibility

public class TestIcebergSparkCompatibility
extends io.trino.tempto.ProductTest

Tests compatibility between Iceberg connector and Spark Iceberg.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static enum	TestIcebergSparkCompatibility.CreateMode
static enum	TestIcebergSparkCompatibility.StorageFormat

Constructor Summary

Constructors

Constructor	Description
TestIcebergSparkCompatibility()

Method Summary

Modifier and Type	Method	Description
void	setup()
void	setUp()
static Object[][]	specVersions()
static Object[][]	storageFormats()
Object[][]	storageFormatsAndCompressionCodecs()
static Object[][]	storageFormatsWithSpecVersion()
static Object[][]	tableFormatWithDeleteFormat()
void	tearDown()
void	testAddNotNullColumn()
void	testAlterTableExecuteProceduresOnEmptyTable()
void	testCleaningUpIcebergTableWithRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat, TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)
void	testCreateAndDropTableWithSameLocationFailsOnTrino(int specVersion)
void	testCreateAndDropTableWithSameLocationWorksOnSpark(int specVersion)
void	testDeleteAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testHandlingPartitionSchemaEvolutionInPartitionMetadata()
void	testIdBasedFieldMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testInsertReadingFromParquetTableWithNestedRowFieldNotPresentInDataFile()
void	testMetadataCompressionCodecGzip()
void	testMigratedDataWithAlteredSchema(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testMigratedDataWithPartialNameMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testMissingMetrics()
void	testOptimizeOnV2IcebergTable()
void	testPartialStats()
void	testPartitionedByNestedField()
void	testPartitioningWithMixedCaseColumnUnsupportedInTrino()
void	testReadAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testRegisterTableWithComments(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testRegisterTableWithDifferentTableName(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testRegisterTableWithDroppedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testRegisterTableWithMetadataFile(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testRegisterTableWithReInsert(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testRegisterTableWithShowCreateTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testRegisterTableWithTableLocation(TestIcebergSparkCompatibility.StorageFormat storageFormat)
static Object[][]	testSetColumnTypeDataProvider()
static Object[][]	testSparkAlterColumnType()
void	testSparkAlterColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat, String sourceColumnType, String sourceValueLiteral, String newColumnType, Object newValue)
void	testSparkAlterPartitionedColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat, String sourceColumnType, String sourceValueLiteral, String newColumnType, Object newValue)
void	testSparkAlterStructColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkCreatesTrinoDrops(int specVersion)
void	testSparkPartitionedByDoubleWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkPartitionedByRealWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadingCompositeTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadingNestedTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadingTrinoCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat, String compressionCodec)
void	testSparkReadingTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat, TestIcebergSparkCompatibility.CreateMode createMode)
Object[][]	testSparkReadingTrinoDataDataProvider()
void	testSparkReadsTrinoPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoTableAfterCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testSparkReadsTrinoTableAfterOptimizeAndCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testSparkReadTrinoUuid(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testStatsAfterAddingPartitionField()
void	testStringPartitioningWithSpecialCharactersCtasInTrino()
void	testStringPartitioningWithSpecialCharactersInsertInSpark()
void	testStringPartitioningWithSpecialCharactersInsertInTrino()
void	testTrinoAlterStructColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testTrinoAnalyze()
void	testTrinoAnalyzeWithNonLowercaseColumnName()
void	testTrinoCreatesSparkDrops()
void	testTrinoPartitionedByDoubleWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testTrinoPartitionedByRealWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testTrinoReadingCompositeSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadingMigratedNestedData(TestIcebergSparkCompatibility.StorageFormat storageFormat)
void	testTrinoReadingNestedSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadingSparkCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat, String compressionCodec)
void	testTrinoReadingSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadingSparkIcebergTablePropertiesData(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadsSparkPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoReadsSparkRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat, TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)
void	testTrinoReadsSparkRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat, TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)
void	testTrinoReadsTrinoTableWithSparkDeletesAfterOptimizeAndCleanUp(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoSetColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat, String sourceColumnType, String sourceValueLiteral, String newColumnType, Object newValue)
void	testTrinoSetPartitionedColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat, String sourceColumnType, String sourceValueLiteral, String newColumnType, Object newValue)
void	testTrinoShowingSparkCreatedTables(int specVersion)
void	testTrinoSparkConcurrentInsert()
void	testTrinoWritingDataWithObjectStorageLocationProvider(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testTrinoWritingDataWithWriterDataPathSet(TestIcebergSparkCompatibility.StorageFormat storageFormat, int specVersion)
void	testUnregisterNotIcebergTable()
void	testUpdateAfterSchemaEvolution()
void	testUpdateOnPartitionColumn()
void	verifyCompressionCodecsDataProvider()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

TestIcebergSparkCompatibility

public TestIcebergSparkCompatibility()

Method Details

setup

public void setup()
           throws org.apache.thrift.TException

Throws:

org.apache.thrift.TException

tearDown

public void tearDown()

setUp

public void setUp()

testTrinoReadingSparkData

public void testTrinoReadingSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testSparkReadingTrinoData

public void testSparkReadingTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 TestIcebergSparkCompatibility.CreateMode createMode)

testSparkReadingTrinoDataDataProvider

public Object[][] testSparkReadingTrinoDataDataProvider()

testSparkReadTrinoUuid

public void testSparkReadTrinoUuid(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkCreatesTrinoDrops

public void testSparkCreatesTrinoDrops(int specVersion)

testTrinoCreatesSparkDrops

public void testTrinoCreatesSparkDrops()

testSparkReadsTrinoPartitionedTable

public void testSparkReadsTrinoPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testTrinoReadsSparkPartitionedTable

public void testTrinoReadsSparkPartitionedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testTrinoPartitionedByRealWithNaN

public void testTrinoPartitionedByRealWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testTrinoPartitionedByDoubleWithNaN

public void testTrinoPartitionedByDoubleWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkPartitionedByRealWithNaN

public void testSparkPartitionedByRealWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkPartitionedByDoubleWithNaN

public void testSparkPartitionedByDoubleWithNaN(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testPartitionedByNestedField

public void testPartitionedByNestedField()

testTrinoReadingCompositeSparkData

public void testTrinoReadingCompositeSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testSparkReadingCompositeTrinoData

public void testSparkReadingCompositeTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testTrinoReadingSparkIcebergTablePropertiesData

public void testTrinoReadingSparkIcebergTablePropertiesData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testTrinoReadingNestedSparkData

public void testTrinoReadingNestedSparkData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testSparkReadingNestedTrinoData

public void testSparkReadingNestedTrinoData(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testIdBasedFieldMapping

public void testIdBasedFieldMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testReadAfterPartitionEvolution

public void testReadAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testTrinoShowingSparkCreatedTables

public void testTrinoShowingSparkCreatedTables(int specVersion)

testCreateAndDropTableWithSameLocationWorksOnSpark

public void testCreateAndDropTableWithSameLocationWorksOnSpark(int specVersion)

testCreateAndDropTableWithSameLocationFailsOnTrino

public void testCreateAndDropTableWithSameLocationFailsOnTrino(int specVersion)

testTrinoWritingDataWithObjectStorageLocationProvider

public void testTrinoWritingDataWithObjectStorageLocationProvider(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testTrinoWritingDataWithWriterDataPathSet

public void testTrinoWritingDataWithWriterDataPathSet(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testStringPartitioningWithSpecialCharactersCtasInTrino

public void testStringPartitioningWithSpecialCharactersCtasInTrino()

testStringPartitioningWithSpecialCharactersInsertInTrino

public void testStringPartitioningWithSpecialCharactersInsertInTrino()

testStringPartitioningWithSpecialCharactersInsertInSpark

public void testStringPartitioningWithSpecialCharactersInsertInSpark()

testPartitioningWithMixedCaseColumnUnsupportedInTrino

public void testPartitioningWithMixedCaseColumnUnsupportedInTrino()

testInsertReadingFromParquetTableWithNestedRowFieldNotPresentInDataFile

public void testInsertReadingFromParquetTableWithNestedRowFieldNotPresentInDataFile()

testTrinoSparkConcurrentInsert

public void testTrinoSparkConcurrentInsert()
                                    throws Exception

Throws:

Exception

See Also:

• TestIcebergInsert.testIcebergConcurrentInsert()

testTrinoReadingSparkCompressedData

public void testTrinoReadingSparkCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 String compressionCodec)

testSparkReadingTrinoCompressedData

public void testSparkReadingTrinoCompressedData(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 String compressionCodec)

verifyCompressionCodecsDataProvider

public void verifyCompressionCodecsDataProvider()

storageFormatsAndCompressionCodecs

public Object[][] storageFormatsAndCompressionCodecs()

testTrinoReadingMigratedNestedData

public void testTrinoReadingMigratedNestedData(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testMigratedDataWithAlteredSchema

public void testMigratedDataWithAlteredSchema(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testMigratedDataWithPartialNameMapping

public void testMigratedDataWithPartialNameMapping(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testPartialStats

public void testPartialStats()

testStatsAfterAddingPartitionField

public void testStatsAfterAddingPartitionField()

testTrinoReadsSparkRowLevelDeletes

public void testTrinoReadsSparkRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat,
 TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)

testTrinoReadsSparkRowLevelDeletesWithRowTypes

public void testTrinoReadsSparkRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat,
 TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)

testSparkReadsTrinoRowLevelDeletes

public void testSparkReadsTrinoRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkReadsTrinoRowLevelDeletesWithRowTypes

public void testSparkReadsTrinoRowLevelDeletesWithRowTypes(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testDeleteAfterPartitionEvolution

public void testDeleteAfterPartitionEvolution(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testMissingMetrics

public void testMissingMetrics()

testOptimizeOnV2IcebergTable

public void testOptimizeOnV2IcebergTable()

testAlterTableExecuteProceduresOnEmptyTable

public void testAlterTableExecuteProceduresOnEmptyTable()

specVersions

public static Object[][] specVersions()

storageFormats

public static Object[][] storageFormats()

tableFormatWithDeleteFormat

public static Object[][] tableFormatWithDeleteFormat()

storageFormatsWithSpecVersion

public static Object[][] storageFormatsWithSpecVersion()

testSparkReadsTrinoTableAfterCleaningUp

public void testSparkReadsTrinoTableAfterCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkReadsTrinoTableAfterOptimizeAndCleaningUp

public void testSparkReadsTrinoTableAfterOptimizeAndCleaningUp(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testTrinoReadsTrinoTableWithSparkDeletesAfterOptimizeAndCleanUp

public void testTrinoReadsTrinoTableWithSparkDeletesAfterOptimizeAndCleanUp(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 int specVersion)

testCleaningUpIcebergTableWithRowLevelDeletes

public void testCleaningUpIcebergTableWithRowLevelDeletes(TestIcebergSparkCompatibility.StorageFormat tableStorageFormat,
 TestIcebergSparkCompatibility.StorageFormat deleteFileStorageFormat)

testUpdateAfterSchemaEvolution

public void testUpdateAfterSchemaEvolution()

testUpdateOnPartitionColumn

public void testUpdateOnPartitionColumn()

testAddNotNullColumn

public void testAddNotNullColumn()

testHandlingPartitionSchemaEvolutionInPartitionMetadata

public void testHandlingPartitionSchemaEvolutionInPartitionMetadata()

testMetadataCompressionCodecGzip

public void testMetadataCompressionCodecGzip()

testTrinoAnalyze

public void testTrinoAnalyze()

testTrinoAnalyzeWithNonLowercaseColumnName

public void testTrinoAnalyzeWithNonLowercaseColumnName()

testRegisterTableWithTableLocation

public void testRegisterTableWithTableLocation(TestIcebergSparkCompatibility.StorageFormat storageFormat)
                                        throws org.apache.thrift.TException

Throws:

org.apache.thrift.TException

testRegisterTableWithComments

public void testRegisterTableWithComments(TestIcebergSparkCompatibility.StorageFormat storageFormat)
                                   throws org.apache.thrift.TException

Throws:

org.apache.thrift.TException

testRegisterTableWithShowCreateTable

public void testRegisterTableWithShowCreateTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)
                                          throws org.apache.thrift.TException

Throws:

org.apache.thrift.TException

testRegisterTableWithReInsert

public void testRegisterTableWithReInsert(TestIcebergSparkCompatibility.StorageFormat storageFormat)
                                   throws org.apache.thrift.TException

Throws:

org.apache.thrift.TException

testRegisterTableWithDroppedTable

public void testRegisterTableWithDroppedTable(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testRegisterTableWithDifferentTableName

public void testRegisterTableWithDifferentTableName(TestIcebergSparkCompatibility.StorageFormat storageFormat)
                                             throws org.apache.thrift.TException

Throws:

org.apache.thrift.TException

testRegisterTableWithMetadataFile

public void testRegisterTableWithMetadataFile(TestIcebergSparkCompatibility.StorageFormat storageFormat)
                                       throws org.apache.thrift.TException

Throws:

org.apache.thrift.TException

testUnregisterNotIcebergTable

public void testUnregisterNotIcebergTable()

testTrinoSetColumnType

public void testTrinoSetColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 String sourceColumnType,
 String sourceValueLiteral,
 String newColumnType,
 Object newValue)

testTrinoSetPartitionedColumnType

public void testTrinoSetPartitionedColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 String sourceColumnType,
 String sourceValueLiteral,
 String newColumnType,
 Object newValue)

testSetColumnTypeDataProvider

public static Object[][] testSetColumnTypeDataProvider()

testTrinoAlterStructColumnType

public void testTrinoAlterStructColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat)

testSparkAlterColumnType

public void testSparkAlterColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 String sourceColumnType,
 String sourceValueLiteral,
 String newColumnType,
 Object newValue)

testSparkAlterPartitionedColumnType

public void testSparkAlterPartitionedColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat,
 String sourceColumnType,
 String sourceValueLiteral,
 String newColumnType,
 Object newValue)

testSparkAlterColumnType

public static Object[][] testSparkAlterColumnType()

testSparkAlterStructColumnType

public void testSparkAlterStructColumnType(TestIcebergSparkCompatibility.StorageFormat storageFormat)