org.apache.hudi.aws.sync

Class AWSGlueCatalogSyncClient

java.lang.Object

org.apache.hudi.sync.common.HoodieSyncClient

org.apache.hudi.aws.sync.AWSGlueCatalogSyncClient

All Implemented Interfaces:

AutoCloseable, org.apache.hudi.sync.common.HoodieMetaSyncOperations

public class AWSGlueCatalogSyncClient
extends org.apache.hudi.sync.common.HoodieSyncClient

This class implements all the AWS APIs to enable syncing of a Hudi Table with the AWS Glue Data Catalog (https://docs.aws.amazon.com/glue/latest/dg/populate-data-catalog.html).

Field Summary

Fields inherited from class org.apache.hudi.sync.common.HoodieSyncClient

config, metaClient, partitionValueExtractor

Fields inherited from interface org.apache.hudi.sync.common.HoodieMetaSyncOperations

HOODIE_LAST_COMMIT_COMPLETION_TIME_SYNC, HOODIE_LAST_COMMIT_TIME_SYNC

Constructor Summary

Constructors

Constructor and Description
AWSGlueCatalogSyncClient(HiveSyncConfig config)

Method Summary

Modifier and Type	Method and Description
void	addPartitionsToTable(String tableName, List<String> partitionsToAdd)
void	close()
void	createDatabase(String databaseName)
void	createTable(String tableName, org.apache.parquet.schema.MessageType storageSchema, String inputFormatClass, String outputFormatClass, String serdeClass, Map<String,String> serdeProperties, Map<String,String> tableProperties)
boolean	databaseExists(String databaseName)
void	deleteLastReplicatedTimeStamp(String tableName)
void	dropPartitions(String tableName, List<String> partitionsToDrop)
List<org.apache.hudi.sync.common.model.Partition>	getAllPartitions(String tableName)
Option<String>	getLastCommitTimeSynced(String tableName)
Option<String>	getLastReplicatedTime(String tableName)
Map<String,String>	getMetastoreSchema(String tableName)
List<org.apache.hudi.sync.common.model.FieldSchema>	getStorageFieldSchemas()
boolean	tableExists(String tableName)
void	updateLastCommitTimeSynced(String tableName)
void	updateLastReplicatedTimeStamp(String tableName, String timeStamp)
void	updatePartitionsToTable(String tableName, List<String> changedPartitions)
boolean	updateTableComments(String tableName, List<org.apache.hudi.sync.common.model.FieldSchema> fromMetastore, List<org.apache.hudi.sync.common.model.FieldSchema> fromStorage)
boolean	updateTableProperties(String tableName, Map<String,String> tableProperties)
void	updateTableSchema(String tableName, org.apache.parquet.schema.MessageType newSchema)

Methods inherited from class org.apache.hudi.sync.common.HoodieSyncClient

getActiveTimeline, getAllPartitionPathsOnStorage, getBasePath, getDroppedPartitionsSince, getMetaClient, getPartitionEvents, getPartitionEvents, getStorageSchema, getStorageSchema, getTableType, getWrittenPartitionsSince, isBootstrap

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.hudi.sync.common.HoodieMetaSyncOperations

dropTable, getLastCommitCompletionTimeSynced, getMetastoreFieldSchemas, getPartitionsByFilter, updateSerdeProperties

Constructor Detail

AWSGlueCatalogSyncClient

public AWSGlueCatalogSyncClient(HiveSyncConfig config)

Method Detail

getAllPartitions

public List<org.apache.hudi.sync.common.model.Partition> getAllPartitions(String tableName)

addPartitionsToTable

public void addPartitionsToTable(String tableName,
                                 List<String> partitionsToAdd)

updatePartitionsToTable

public void updatePartitionsToTable(String tableName,
                                    List<String> changedPartitions)

dropPartitions

public void dropPartitions(String tableName,
                           List<String> partitionsToDrop)

updateTableProperties

public boolean updateTableProperties(String tableName,
                                     Map<String,String> tableProperties)

getStorageFieldSchemas

public List<org.apache.hudi.sync.common.model.FieldSchema> getStorageFieldSchemas()

updateTableComments

public boolean updateTableComments(String tableName,
                                   List<org.apache.hudi.sync.common.model.FieldSchema> fromMetastore,
                                   List<org.apache.hudi.sync.common.model.FieldSchema> fromStorage)

updateTableSchema

public void updateTableSchema(String tableName,
                              org.apache.parquet.schema.MessageType newSchema)

createTable

public void createTable(String tableName,
                        org.apache.parquet.schema.MessageType storageSchema,
                        String inputFormatClass,
                        String outputFormatClass,
                        String serdeClass,
                        Map<String,String> serdeProperties,
                        Map<String,String> tableProperties)

getMetastoreSchema

public Map<String,String> getMetastoreSchema(String tableName)

tableExists

public boolean tableExists(String tableName)

databaseExists

public boolean databaseExists(String databaseName)

createDatabase

public void createDatabase(String databaseName)

getLastCommitTimeSynced

public Option<String> getLastCommitTimeSynced(String tableName)

close

public void close()

updateLastCommitTimeSynced

public void updateLastCommitTimeSynced(String tableName)

getLastReplicatedTime

public Option<String> getLastReplicatedTime(String tableName)

updateLastReplicatedTimeStamp

public void updateLastReplicatedTimeStamp(String tableName,
                                          String timeStamp)

deleteLastReplicatedTimeStamp

public void deleteLastReplicatedTimeStamp(String tableName)