org.apache.hudi.storage.hadoop

Class HoodieHadoopStorage

java.lang.Object

org.apache.hudi.storage.HoodieStorage

org.apache.hudi.storage.hadoop.HoodieHadoopStorage

All Implemented Interfaces:

Closeable, AutoCloseable

public class HoodieHadoopStorage
extends HoodieStorage

Implementation of HoodieStorage using Hadoop's FileSystem

Field Summary

Fields inherited from class org.apache.hudi.storage.HoodieStorage

LOG, storageConf

Constructor Summary

Constructors

Constructor and Description
HoodieHadoopStorage(org.apache.hadoop.fs.FileSystem fs)
HoodieHadoopStorage(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf)
HoodieHadoopStorage(StoragePath path, StorageConfiguration<?> conf)
HoodieHadoopStorage(StoragePath path, StorageConfiguration<?> conf, boolean enableRetry, long maxRetryIntervalMs, int maxRetryNumbers, long initialRetryIntervalMs, String retryExceptions, ConsistencyGuard consistencyGuard)
HoodieHadoopStorage(String path, org.apache.hadoop.conf.Configuration conf)
HoodieHadoopStorage(String path, StorageConfiguration<?> conf)

Method Summary

Modifier and Type	Method and Description
OutputStream	append(StoragePath path)
void	close()
OutputStream	create(StoragePath path)
OutputStream	create(StoragePath path, boolean overwrite)
OutputStream	create(StoragePath path, boolean overwrite, Integer bufferSize, Short replication, Long sizeThreshold)
boolean	createDirectory(StoragePath path)
boolean	createNewFile(StoragePath path)
boolean	deleteDirectory(StoragePath path)
boolean	deleteFile(StoragePath path)
boolean	exists(StoragePath path)
int	getDefaultBlockSize(StoragePath path)
int	getDefaultBufferSize()
short	getDefaultReplication(StoragePath path)
Object	getFileSystem()
StoragePathInfo	getPathInfo(StoragePath path)
HoodieStorage	getRawStorage()
String	getScheme()
URI	getUri()
List<StoragePathInfo>	globEntries(StoragePath pathPattern)
List<StoragePathInfo>	globEntries(StoragePath pathPattern, StoragePathFilter filter)
List<StoragePathInfo>	listDirectEntries(List<StoragePath> pathList)
List<StoragePathInfo>	listDirectEntries(StoragePath path)
List<StoragePathInfo>	listDirectEntries(StoragePath path, StoragePathFilter filter)
List<StoragePathInfo>	listFiles(StoragePath path)
HoodieStorage	newInstance(StoragePath path, StorageConfiguration<?> storageConf)
InputStream	open(StoragePath path)
SeekableDataInputStream	openSeekable(StoragePath path, int bufferSize, boolean wrapStream)
boolean	rename(StoragePath oldPath, StoragePath newPath)

Methods inherited from class org.apache.hudi.storage.HoodieStorage

createImmutableFileInPath, getConf, needCreateTempFile, openSeekable

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HoodieHadoopStorage

public HoodieHadoopStorage(StoragePath path,
                           StorageConfiguration<?> conf)

HoodieHadoopStorage

public HoodieHadoopStorage(org.apache.hadoop.fs.Path path,
                           org.apache.hadoop.conf.Configuration conf)

HoodieHadoopStorage

public HoodieHadoopStorage(String path,
                           org.apache.hadoop.conf.Configuration conf)

HoodieHadoopStorage

public HoodieHadoopStorage(String path,
                           StorageConfiguration<?> conf)

HoodieHadoopStorage

public HoodieHadoopStorage(StoragePath path,
                           StorageConfiguration<?> conf,
                           boolean enableRetry,
                           long maxRetryIntervalMs,
                           int maxRetryNumbers,
                           long initialRetryIntervalMs,
                           String retryExceptions,
                           ConsistencyGuard consistencyGuard)

HoodieHadoopStorage

public HoodieHadoopStorage(org.apache.hadoop.fs.FileSystem fs)

Method Detail

newInstance

public HoodieStorage newInstance(StoragePath path,
                                 StorageConfiguration<?> storageConf)

Specified by:

newInstance in class HoodieStorage

getScheme

public String getScheme()

Specified by:

getScheme in class HoodieStorage

getUri

public URI getUri()

Specified by:

getUri in class HoodieStorage

getDefaultBlockSize

public int getDefaultBlockSize(StoragePath path)

Specified by:

getDefaultBlockSize in class HoodieStorage

create

public OutputStream create(StoragePath path,
                           boolean overwrite)
                    throws IOException

Specified by:

create in class HoodieStorage

Throws:

IOException

create

public OutputStream create(StoragePath path,
                           boolean overwrite,
                           Integer bufferSize,
                           Short replication,
                           Long sizeThreshold)
                    throws IOException

Specified by:

create in class HoodieStorage

Throws:

IOException

getDefaultBufferSize

public int getDefaultBufferSize()

Specified by:

getDefaultBufferSize in class HoodieStorage

getDefaultReplication

public short getDefaultReplication(StoragePath path)

Specified by:

getDefaultReplication in class HoodieStorage

open

public InputStream open(StoragePath path)
                 throws IOException

Specified by:

open in class HoodieStorage

Throws:

IOException

openSeekable

public SeekableDataInputStream openSeekable(StoragePath path,
                                            int bufferSize,
                                            boolean wrapStream)
                                     throws IOException

Specified by:

openSeekable in class HoodieStorage

Throws:

IOException

append

public OutputStream append(StoragePath path)
                    throws IOException

Specified by:

append in class HoodieStorage

Throws:

IOException

exists

public boolean exists(StoragePath path)
               throws IOException

Specified by:

exists in class HoodieStorage

Throws:

IOException

getPathInfo

public StoragePathInfo getPathInfo(StoragePath path)
                            throws IOException

Specified by:

getPathInfo in class HoodieStorage

Throws:

IOException

createDirectory

public boolean createDirectory(StoragePath path)
                        throws IOException

Specified by:

createDirectory in class HoodieStorage

Throws:

IOException

listDirectEntries

public List<StoragePathInfo> listDirectEntries(StoragePath path)
                                        throws IOException

Specified by:

listDirectEntries in class HoodieStorage

Throws:

IOException

listFiles

public List<StoragePathInfo> listFiles(StoragePath path)
                                throws IOException

Specified by:

listFiles in class HoodieStorage

Throws:

IOException

listDirectEntries

public List<StoragePathInfo> listDirectEntries(List<StoragePath> pathList)
                                        throws IOException

Overrides:

listDirectEntries in class HoodieStorage

Throws:

IOException

listDirectEntries

public List<StoragePathInfo> listDirectEntries(StoragePath path,
                                               StoragePathFilter filter)
                                        throws IOException

Specified by:

listDirectEntries in class HoodieStorage

Throws:

IOException

globEntries

public List<StoragePathInfo> globEntries(StoragePath pathPattern)
                                  throws IOException

Overrides:

globEntries in class HoodieStorage

Throws:

IOException

globEntries

public List<StoragePathInfo> globEntries(StoragePath pathPattern,
                                         StoragePathFilter filter)
                                  throws IOException

Specified by:

globEntries in class HoodieStorage

Throws:

IOException

rename

public boolean rename(StoragePath oldPath,
                      StoragePath newPath)
               throws IOException

Specified by:

rename in class HoodieStorage

Throws:

IOException

deleteDirectory

public boolean deleteDirectory(StoragePath path)
                        throws IOException

Specified by:

deleteDirectory in class HoodieStorage

Throws:

IOException

deleteFile

public boolean deleteFile(StoragePath path)
                   throws IOException

Specified by:

deleteFile in class HoodieStorage

Throws:

IOException

getFileSystem

public Object getFileSystem()

Specified by:

getFileSystem in class HoodieStorage

getRawStorage

public HoodieStorage getRawStorage()

Specified by:

getRawStorage in class HoodieStorage

create

public OutputStream create(StoragePath path)
                    throws IOException

Overrides:

create in class HoodieStorage

Throws:

IOException

createNewFile

public boolean createNewFile(StoragePath path)
                      throws IOException

Overrides:

createNewFile in class HoodieStorage

Throws:

IOException

close

public void close()
           throws IOException

Throws:

IOException