org.apache.hudi.hadoop.fs

Class HoodieRetryWrapperFileSystem

java.lang.Object

org.apache.hadoop.conf.Configured

org.apache.hadoop.fs.FileSystem

org.apache.hudi.hadoop.fs.HoodieRetryWrapperFileSystem

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.conf.Configurable

public class HoodieRetryWrapperFileSystem
extends org.apache.hadoop.fs.FileSystem

Extending FileSystem implementation of I/O operations with retries.

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FileSystem.DirectoryEntries, org.apache.hadoop.fs.FileSystem.Statistics

Field Summary

Fields inherited from class org.apache.hadoop.fs.FileSystem

DEFAULT_FS, FS_DEFAULT_NAME_KEY, LOG, SHUTDOWN_HOOK_PRIORITY, statistics, TRASH_PREFIX, USER_HOME_PREFIX

Constructor Summary

Constructors

Constructor and Description
HoodieRetryWrapperFileSystem(org.apache.hadoop.fs.FileSystem fs, long maxRetryIntervalMs, int maxRetryNumbers, long initialRetryIntervalMs, String retryExceptions)

Method Summary

Modifier and Type	Method and Description
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f, int bufferSize)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f, int bufferSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress, org.apache.hadoop.fs.Options.ChecksumOpt checksumOpt)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, short replication)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, short replication, org.apache.hadoop.util.Progressable progress)
boolean	createNewFile(org.apache.hadoop.fs.Path f)
boolean	delete(org.apache.hadoop.fs.Path f)
boolean	delete(org.apache.hadoop.fs.Path f, boolean recursive)
boolean	exists(org.apache.hadoop.fs.Path f)
org.apache.hadoop.conf.Configuration	getConf()
short	getDefaultReplication()
short	getDefaultReplication(org.apache.hadoop.fs.Path path)
org.apache.hadoop.fs.FileStatus	getFileStatus(org.apache.hadoop.fs.Path f)
String	getScheme()
URI	getUri()
org.apache.hadoop.fs.Path	getWorkingDirectory()
org.apache.hadoop.fs.FileStatus[]	globStatus(org.apache.hadoop.fs.Path pathPattern)
org.apache.hadoop.fs.FileStatus[]	globStatus(org.apache.hadoop.fs.Path pathPattern, org.apache.hadoop.fs.PathFilter filter)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus>	listFiles(org.apache.hadoop.fs.Path f, boolean recursive)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus>	listLocatedStatus(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path[] files)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path[] files, org.apache.hadoop.fs.PathFilter filter)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.PathFilter filter)
boolean	mkdirs(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path f, int bufferSize)
boolean	rename(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	setWorkingDirectory(org.apache.hadoop.fs.Path newDir)

Methods inherited from class org.apache.hadoop.fs.FileSystem

access, addDelegationTokens, appendFile, areSymlinksEnabled, cancelDeleteOnExit, canonicalizeUri, checkPath, clearStatistics, close, closeAll, closeAllForUGI, completeLocalOutput, concat, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyToLocalFile, copyToLocalFile, copyToLocalFile, create, createFile, createNonRecursive, createNonRecursive, createNonRecursive, createSnapshot, createSnapshot, createSymlink, deleteOnExit, deleteSnapshot, enableSymlinks, fixRelativePart, get, get, get, getAclStatus, getAllStatistics, getAllStoragePolicies, getBlockSize, getCanonicalServiceName, getCanonicalUri, getChildFileSystems, getContentSummary, getDefaultBlockSize, getDefaultBlockSize, getDefaultPort, getDefaultUri, getDelegationToken, getFileBlockLocations, getFileBlockLocations, getFileChecksum, getFileChecksum, getFileLinkStatus, getFileSystemClass, getFSofPath, getGlobalStorageStatistics, getHomeDirectory, getInitialWorkingDirectory, getLength, getLinkTarget, getLocal, getName, getNamed, getQuotaUsage, getReplication, getServerDefaults, getServerDefaults, getStatistics, getStatistics, getStatus, getStatus, getStoragePolicy, getStorageStatistics, getTrashRoot, getTrashRoots, getUsed, getUsed, getXAttr, getXAttrs, getXAttrs, initialize, isDirectory, isFile, listCorruptFileBlocks, listLocatedStatus, listStatusBatch, listStatusIterator, listXAttrs, makeQualified, mkdirs, mkdirs, modifyAclEntries, moveFromLocalFile, moveFromLocalFile, moveToLocalFile, msync, newInstance, newInstance, newInstance, newInstanceLocal, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, removeAcl, removeAclEntries, removeDefaultAcl, removeXAttr, rename, renameSnapshot, resolveLink, resolvePath, setAcl, setDefaultUri, setDefaultUri, setOwner, setPermission, setReplication, setStoragePolicy, setTimes, setVerifyChecksum, setWriteChecksum, setXAttr, setXAttr, startLocalOutput, supportsSymlinks, truncate, unsetStoragePolicy

Methods inherited from class org.apache.hadoop.conf.Configured

setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HoodieRetryWrapperFileSystem

public HoodieRetryWrapperFileSystem(org.apache.hadoop.fs.FileSystem fs,
                                    long maxRetryIntervalMs,
                                    int maxRetryNumbers,
                                    long initialRetryIntervalMs,
                                    String retryExceptions)

Method Detail

getUri

public URI getUri()

Specified by:

getUri in class org.apache.hadoop.fs.FileSystem

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f,
                                                   int bufferSize)
                                            throws IOException

Specified by:

open in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f)
                                            throws IOException

Overrides:

open in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Specified by:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      short replication)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      short replication,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress,
                                                      org.apache.hadoop.fs.Options.ChecksumOpt checksumOpt)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createNewFile

public boolean createNewFile(org.apache.hadoop.fs.Path f)
                      throws IOException

Overrides:

createNewFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f,
                                                      int bufferSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Specified by:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f)
                                               throws IOException

Overrides:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f,
                                                      int bufferSize)
                                               throws IOException

Overrides:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

rename

public boolean rename(org.apache.hadoop.fs.Path src,
                      org.apache.hadoop.fs.Path dst)
               throws IOException

Specified by:

rename in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

delete

public boolean delete(org.apache.hadoop.fs.Path f,
                      boolean recursive)
               throws IOException

Specified by:

delete in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

delete

public boolean delete(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

delete in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path f)
                                             throws FileNotFoundException,
                                                    IOException

Specified by:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

FileNotFoundException

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path f,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Overrides:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path[] files)
                                             throws IOException

Overrides:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path[] files,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Overrides:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

globStatus

public org.apache.hadoop.fs.FileStatus[] globStatus(org.apache.hadoop.fs.Path pathPattern)
                                             throws IOException

Overrides:

globStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

globStatus

public org.apache.hadoop.fs.FileStatus[] globStatus(org.apache.hadoop.fs.Path pathPattern,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Overrides:

globStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listLocatedStatus

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> listLocatedStatus(org.apache.hadoop.fs.Path f)
                                                                                              throws IOException

Overrides:

listLocatedStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listFiles

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> listFiles(org.apache.hadoop.fs.Path f,
                                                                                             boolean recursive)
                                                                                      throws IOException

Overrides:

listFiles in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setWorkingDirectory

public void setWorkingDirectory(org.apache.hadoop.fs.Path newDir)

Specified by:

setWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getWorkingDirectory

public org.apache.hadoop.fs.Path getWorkingDirectory()

Specified by:

getWorkingDirectory in class org.apache.hadoop.fs.FileSystem

mkdirs

public boolean mkdirs(org.apache.hadoop.fs.Path f,
                      org.apache.hadoop.fs.permission.FsPermission permission)
               throws IOException

Specified by:

mkdirs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getFileStatus

public org.apache.hadoop.fs.FileStatus getFileStatus(org.apache.hadoop.fs.Path f)
                                              throws IOException

Specified by:

getFileStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

exists

public boolean exists(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

exists in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getConf

public org.apache.hadoop.conf.Configuration getConf()

Specified by:

getConf in interface org.apache.hadoop.conf.Configurable

Overrides:

getConf in class org.apache.hadoop.conf.Configured

getScheme

public String getScheme()

Overrides:

getScheme in class org.apache.hadoop.fs.FileSystem

getDefaultReplication

public short getDefaultReplication()

Overrides:

getDefaultReplication in class org.apache.hadoop.fs.FileSystem

getDefaultReplication

public short getDefaultReplication(org.apache.hadoop.fs.Path path)

Overrides:

getDefaultReplication in class org.apache.hadoop.fs.FileSystem