Package io.trino.plugin.hive.metastore

Class SemiTransactionalHiveMetastore

java.lang.Object

io.trino.plugin.hive.metastore.SemiTransactionalHiveMetastore

All Implemented Interfaces:

SqlStandardAccessControlMetadataMetastore

public class SemiTransactionalHiveMetastore
extends Object
implements SqlStandardAccessControlMetadataMetastore

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	SemiTransactionalHiveMetastore.Action<T>
static final record	SemiTransactionalHiveMetastore.PartitionUpdateInfo

Constructor Summary

Constructors

Constructor

Description

SemiTransactionalHiveMetastore(HdfsEnvironment hdfsEnvironment, HiveMetastoreClosure delegate, Executor fileSystemExecutor, Executor dropExecutor, Executor updateExecutor, boolean skipDeletionForAlter, boolean skipTargetCleanupOnRollback, boolean deleteSchemaLocationsFallback, Optional<io.airlift.units.Duration> hiveTransactionHeartbeatInterval, ScheduledExecutorService heartbeatService, TableInvalidationCallback tableInvalidationCallback)

Method Summary

Modifier and Type	Method	Description
void	addColumn(String databaseName, String tableName, String columnName, HiveType columnType, String columnComment)
void	addDynamicPartitions(String dbName, String tableName, List<String> partitionNames, long transactionId, long writeId, AcidOperation operation)
void	addPartition(ConnectorSession session, String databaseName, String tableName, Partition partition, Location currentLocation, Optional<List<String>> files, PartitionStatistics statistics, boolean cleanExtraOutputFilesOnCommit)
void	alterPartitions(String dbName, String tableName, List<Partition> partitions, long writeId)
AcidTransaction	beginInsert(ConnectorSession session, Table table)
AcidTransaction	beginMerge(ConnectorSession session, Table table)
void	beginQuery(ConnectorSession session)
void	checkSupportsHiveAcidTransactions()
static void	cleanExtraOutputFiles(HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, String queryId, Location location, Set<String> filesToKeep)
void	cleanupQuery(ConnectorSession session)
void	commentColumn(String databaseName, String tableName, String columnName, Optional<String> comment)
void	commentTable(String databaseName, String tableName, Optional<String> comment)
void	commit()
void	commitTransaction(long transactionId)
void	createDatabase(ConnectorSession session, Database database)
void	createRole(String role, String grantor)
void	createTable(ConnectorSession session, Table table, PrincipalPrivileges principalPrivileges, Optional<org.apache.hadoop.fs.Path> currentPath, Optional<List<String>> files, boolean ignoreExisting, PartitionStatistics statistics, boolean cleanExtraOutputFilesOnCommit)	currentLocation needs to be supplied if a writePath exists for the table.
String	declareIntentionToWrite(ConnectorSession session, LocationHandle.WriteMode writeMode, Location stagingPathRoot, SchemaTableName schemaTableName)
void	dropColumn(String databaseName, String tableName, String columnName)
void	dropDatabase(ConnectorSession session, String schemaName)
void	dropDeclaredIntentionToWrite(String declarationId)
void	dropPartition(ConnectorSession session, String databaseName, String tableName, List<String> partitionValues, boolean deleteData)
void	dropRole(String role)
void	dropTable(ConnectorSession session, String databaseName, String tableName)
void	finishChangingExistingTable(AcidOperation acidOperation, ConnectorSession session, String databaseName, String tableName, Location currentLocation, List<String> fileNames, PartitionStatistics statisticsUpdate, boolean cleanExtraOutputFilesOnCommit)
void	finishInsertIntoExistingPartitions(ConnectorSession session, String databaseName, String tableName, List<SemiTransactionalHiveMetastore.PartitionUpdateInfo> partitionUpdateInfos, boolean cleanExtraOutputFilesOnCommit)
void	finishMerge(ConnectorSession session, String databaseName, String tableName, Location currentLocation, List<PartitionUpdateAndMergeResults> partitionUpdateAndMergeResults, List<Partition> partitions)
HivePageSinkMetadata	generatePageSinkMetadata(SchemaTableName schemaTableName)
List<String>	getAllDatabases()
Optional<List<SchemaTableName>>	getAllTables()
List<String>	getAllTables(String databaseName)
Optional<List<SchemaTableName>>	getAllViews()
List<String>	getAllViews(String databaseName)
Optional<Database>	getDatabase(String databaseName)
Optional<HivePrincipal>	getDatabaseOwner(String databaseName)
Optional<List<String>>	getPartitionNames(String databaseName, String tableName)
Optional<List<String>>	getPartitionNamesByFilter(String databaseName, String tableName, List<String> columnNames, TupleDomain<String> partitionKeysFilter)
Map<String,Optional<Partition>>	getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames)
Map<String,PartitionStatistics>	getPartitionStatistics(String databaseName, String tableName, Set<String> columns, Set<String> partitionNames)
Set<HiveColumnStatisticType>	getSupportedColumnStatistics(Type type)
Optional<Table>	getTable(String databaseName, String tableName)
PartitionStatistics	getTableStatistics(String databaseName, String tableName, Optional<Set<String>> columns)
Optional<ValidTxnWriteIdList>	getValidWriteIds(ConnectorSession session, HiveTableHandle tableHandle)
void	grantRoles(Set<String> roles, Set<HivePrincipal> grantees, boolean adminOption, HivePrincipal grantor)
void	grantTablePrivileges(String databaseName, String tableName, HivePrincipal grantee, HivePrincipal grantor, Set<HivePrivilegeInfo.HivePrivilege> privileges, boolean grantOption)
boolean	isFinished()
boolean	isReadableWithinTransaction(String databaseName, String tableName)
Set<RoleGrant>	listGrantedPrincipals(String role)
Set<RoleGrant>	listRoleGrants(HivePrincipal principal)
Set<String>	listRoles()
Set<HivePrivilegeInfo>	listTablePrivileges(String databaseName, String tableName, Optional<HivePrincipal> principal)
void	renameColumn(String databaseName, String tableName, String oldColumnName, String newColumnName)
void	renameDatabase(String source, String target)
void	renameTable(String databaseName, String tableName, String newDatabaseName, String newTableName)
void	replaceTable(String databaseName, String tableName, Table table, PrincipalPrivileges principalPrivileges)
void	revokeRoles(Set<String> roles, Set<HivePrincipal> grantees, boolean adminOption, HivePrincipal grantor)
void	revokeTablePrivileges(String databaseName, String tableName, HivePrincipal grantee, HivePrincipal grantor, Set<HivePrivilegeInfo.HivePrivilege> privileges, boolean grantOption)
void	rollback()
void	setDatabaseOwner(String source, HivePrincipal principal)
void	setPartitionStatistics(Table table, Map<List<String>,PartitionStatistics> partitionStatisticsMap)
void	setTableOwner(String schema, String table, HivePrincipal principal)
void	setTableStatistics(Table table, PartitionStatistics tableStatistics)
boolean	shouldDeleteDatabaseData(ConnectorSession session, String schemaName)
void	testOnlyCheckIsReadOnly()
void	testOnlyThrowOnCleanupFailures()
void	truncateUnpartitionedTable(ConnectorSession session, String databaseName, String tableName)
HiveMetastoreClosure	unsafeGetRawHiveMetastoreClosure()	Get the underlying metastore closure.
void	updateTableWriteId(String dbName, String tableName, long transactionId, long writeId, OptionalLong rowCountChange)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

SemiTransactionalHiveMetastore

public SemiTransactionalHiveMetastore(HdfsEnvironment hdfsEnvironment,
 HiveMetastoreClosure delegate,
 Executor fileSystemExecutor,
 Executor dropExecutor,
 Executor updateExecutor,
 boolean skipDeletionForAlter,
 boolean skipTargetCleanupOnRollback,
 boolean deleteSchemaLocationsFallback,
 Optional<io.airlift.units.Duration> hiveTransactionHeartbeatInterval,
 ScheduledExecutorService heartbeatService,
 TableInvalidationCallback tableInvalidationCallback)

Method Details

getAllDatabases

public List<String> getAllDatabases()

unsafeGetRawHiveMetastoreClosure

public HiveMetastoreClosure unsafeGetRawHiveMetastoreClosure()

Get the underlying metastore closure. Use this method with caution as it bypasses the current transactional state, so modifications made in the transaction are visible.

getDatabase

public Optional<Database> getDatabase(String databaseName)

getAllTables

public List<String> getAllTables(String databaseName)

getAllTables

public Optional<List<SchemaTableName>> getAllTables()

getTable

public Optional<Table> getTable(String databaseName,
 String tableName)

isReadableWithinTransaction

public boolean isReadableWithinTransaction(String databaseName,
 String tableName)

getSupportedColumnStatistics

public Set<HiveColumnStatisticType> getSupportedColumnStatistics(Type type)

getTableStatistics

public PartitionStatistics getTableStatistics(String databaseName,
 String tableName,
 Optional<Set<String>> columns)

getPartitionStatistics

public Map<String,PartitionStatistics> getPartitionStatistics(String databaseName,
 String tableName,
 Set<String> columns,
 Set<String> partitionNames)

generatePageSinkMetadata

public HivePageSinkMetadata generatePageSinkMetadata(SchemaTableName schemaTableName)

getAllViews

public List<String> getAllViews(String databaseName)

getAllViews

public Optional<List<SchemaTableName>> getAllViews()

createDatabase

public void createDatabase(ConnectorSession session,
 Database database)

dropDatabase

public void dropDatabase(ConnectorSession session,
 String schemaName)

shouldDeleteDatabaseData

public boolean shouldDeleteDatabaseData(ConnectorSession session,
 String schemaName)

renameDatabase

public void renameDatabase(String source,
 String target)

setDatabaseOwner

public void setDatabaseOwner(String source,
 HivePrincipal principal)

setTableStatistics

public void setTableStatistics(Table table,
 PartitionStatistics tableStatistics)

setPartitionStatistics

public void setPartitionStatistics(Table table,
 Map<List<String>,PartitionStatistics> partitionStatisticsMap)

createTable

public void createTable(ConnectorSession session,
 Table table,
 PrincipalPrivileges principalPrivileges,
 Optional<org.apache.hadoop.fs.Path> currentPath,
 Optional<List<String>> files,
 boolean ignoreExisting,
 PartitionStatistics statistics,
 boolean cleanExtraOutputFilesOnCommit)

currentLocation needs to be supplied if a writePath exists for the table.

dropTable

public void dropTable(ConnectorSession session,
 String databaseName,
 String tableName)

replaceTable

public void replaceTable(String databaseName,
 String tableName,
 Table table,
 PrincipalPrivileges principalPrivileges)

renameTable

public void renameTable(String databaseName,
 String tableName,
 String newDatabaseName,
 String newTableName)

commentTable

public void commentTable(String databaseName,
 String tableName,
 Optional<String> comment)

setTableOwner

public void setTableOwner(String schema,
 String table,
 HivePrincipal principal)

commentColumn

public void commentColumn(String databaseName,
 String tableName,
 String columnName,
 Optional<String> comment)

addColumn

public void addColumn(String databaseName,
 String tableName,
 String columnName,
 HiveType columnType,
 String columnComment)

renameColumn

public void renameColumn(String databaseName,
 String tableName,
 String oldColumnName,
 String newColumnName)

dropColumn

public void dropColumn(String databaseName,
 String tableName,
 String columnName)

finishChangingExistingTable

public void finishChangingExistingTable(AcidOperation acidOperation,
 ConnectorSession session,
 String databaseName,
 String tableName,
 Location currentLocation,
 List<String> fileNames,
 PartitionStatistics statisticsUpdate,
 boolean cleanExtraOutputFilesOnCommit)

truncateUnpartitionedTable

public void truncateUnpartitionedTable(ConnectorSession session,
 String databaseName,
 String tableName)

finishMerge

public void finishMerge(ConnectorSession session,
 String databaseName,
 String tableName,
 Location currentLocation,
 List<PartitionUpdateAndMergeResults> partitionUpdateAndMergeResults,
 List<Partition> partitions)

getPartitionNames

public Optional<List<String>> getPartitionNames(String databaseName,
 String tableName)

getPartitionNamesByFilter

public Optional<List<String>> getPartitionNamesByFilter(String databaseName,
 String tableName,
 List<String> columnNames,
 TupleDomain<String> partitionKeysFilter)

getPartitionsByNames

public Map<String,Optional<Partition>> getPartitionsByNames(String databaseName,
 String tableName,
 List<String> partitionNames)

addPartition

public void addPartition(ConnectorSession session,
 String databaseName,
 String tableName,
 Partition partition,
 Location currentLocation,
 Optional<List<String>> files,
 PartitionStatistics statistics,
 boolean cleanExtraOutputFilesOnCommit)

dropPartition

public void dropPartition(ConnectorSession session,
 String databaseName,
 String tableName,
 List<String> partitionValues,
 boolean deleteData)

finishInsertIntoExistingPartitions

public void finishInsertIntoExistingPartitions(ConnectorSession session,
 String databaseName,
 String tableName,
 List<SemiTransactionalHiveMetastore.PartitionUpdateInfo> partitionUpdateInfos,
 boolean cleanExtraOutputFilesOnCommit)

createRole

public void createRole(String role,
 String grantor)

Specified by:

createRole in interface SqlStandardAccessControlMetadataMetastore

dropRole

public void dropRole(String role)

Specified by:

dropRole in interface SqlStandardAccessControlMetadataMetastore

listRoles

public Set<String> listRoles()

Specified by:

listRoles in interface SqlStandardAccessControlMetadataMetastore

grantRoles

public void grantRoles(Set<String> roles,
 Set<HivePrincipal> grantees,
 boolean adminOption,
 HivePrincipal grantor)

Specified by:

grantRoles in interface SqlStandardAccessControlMetadataMetastore

revokeRoles

public void revokeRoles(Set<String> roles,
 Set<HivePrincipal> grantees,
 boolean adminOption,
 HivePrincipal grantor)

Specified by:

revokeRoles in interface SqlStandardAccessControlMetadataMetastore

listGrantedPrincipals

public Set<RoleGrant> listGrantedPrincipals(String role)

Specified by:

listGrantedPrincipals in interface SqlStandardAccessControlMetadataMetastore

listRoleGrants

public Set<RoleGrant> listRoleGrants(HivePrincipal principal)

Specified by:

listRoleGrants in interface SqlStandardAccessControlMetadataMetastore

getDatabaseOwner

public Optional<HivePrincipal> getDatabaseOwner(String databaseName)

Specified by:

getDatabaseOwner in interface SqlStandardAccessControlMetadataMetastore

listTablePrivileges

public Set<HivePrivilegeInfo> listTablePrivileges(String databaseName,
 String tableName,
 Optional<HivePrincipal> principal)

Specified by:

listTablePrivileges in interface SqlStandardAccessControlMetadataMetastore

grantTablePrivileges

public void grantTablePrivileges(String databaseName,
 String tableName,
 HivePrincipal grantee,
 HivePrincipal grantor,
 Set<HivePrivilegeInfo.HivePrivilege> privileges,
 boolean grantOption)

Specified by:

grantTablePrivileges in interface SqlStandardAccessControlMetadataMetastore

revokeTablePrivileges

public void revokeTablePrivileges(String databaseName,
 String tableName,
 HivePrincipal grantee,
 HivePrincipal grantor,
 Set<HivePrivilegeInfo.HivePrivilege> privileges,
 boolean grantOption)

Specified by:

revokeTablePrivileges in interface SqlStandardAccessControlMetadataMetastore

declareIntentionToWrite

public String declareIntentionToWrite(ConnectorSession session,
 LocationHandle.WriteMode writeMode,
 Location stagingPathRoot,
 SchemaTableName schemaTableName)

dropDeclaredIntentionToWrite

public void dropDeclaredIntentionToWrite(String declarationId)

isFinished

public boolean isFinished()

commit

public void commit()

rollback

public void rollback()

checkSupportsHiveAcidTransactions

public void checkSupportsHiveAcidTransactions()

beginQuery

public void beginQuery(ConnectorSession session)

beginInsert

public AcidTransaction beginInsert(ConnectorSession session,
 Table table)

beginMerge

public AcidTransaction beginMerge(ConnectorSession session,
 Table table)

getValidWriteIds

public Optional<ValidTxnWriteIdList> getValidWriteIds(ConnectorSession session,
 HiveTableHandle tableHandle)

cleanupQuery

public void cleanupQuery(ConnectorSession session)

testOnlyCheckIsReadOnly

public void testOnlyCheckIsReadOnly()

testOnlyThrowOnCleanupFailures

public void testOnlyThrowOnCleanupFailures()

updateTableWriteId

public void updateTableWriteId(String dbName,
 String tableName,
 long transactionId,
 long writeId,
 OptionalLong rowCountChange)

alterPartitions

public void alterPartitions(String dbName,
 String tableName,
 List<Partition> partitions,
 long writeId)

addDynamicPartitions

public void addDynamicPartitions(String dbName,
 String tableName,
 List<String> partitionNames,
 long transactionId,
 long writeId,
 AcidOperation operation)

commitTransaction

public void commitTransaction(long transactionId)

cleanExtraOutputFiles

public static void cleanExtraOutputFiles(HdfsEnvironment hdfsEnvironment,
 HdfsContext hdfsContext,
 String queryId,
 Location location,
 Set<String> filesToKeep)