org.apache.hudi.common.fs.inline

Class InLineFileSystem

java.lang.Object

org.apache.hadoop.conf.Configured

org.apache.hadoop.fs.FileSystem

org.apache.hudi.common.fs.inline.InLineFileSystem

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.conf.Configurable

public class InLineFileSystem
extends org.apache.hadoop.fs.FileSystem

Enables reading any inline file at a given offset and length. This FileSystem is used only in read path and does not support any write apis.

- Reading an inlined file at a given offset, length, read it out as if it were an independent file of that length - Inlined path is of the form "inlinefs:///path/to/outer/file//?start_offset=&length=

TODO: The reader/writer may try to use relative paths based on the inlinepath and it may not work. Need to handle this gracefully eg. the parquet summary metadata reading. TODO: If this shows promise, also support directly writing the inlined file to the underneath file without buffer

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FileSystem.DirectoryEntries, org.apache.hadoop.fs.FileSystem.Statistics

Field Summary

Fields

Modifier and Type	Field and Description
static String	SCHEME

Fields inherited from class org.apache.hadoop.fs.FileSystem

DEFAULT_FS, FS_DEFAULT_NAME_KEY, LOG, SHUTDOWN_HOOK_PRIORITY, statistics, TRASH_PREFIX, USER_HOME_PREFIX

Constructor Summary

Constructors

Constructor and Description
InLineFileSystem()

Method Summary

Modifier and Type	Method and Description
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path path, int i, org.apache.hadoop.util.Progressable progressable)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission fsPermission, boolean b, int i, short i1, long l, org.apache.hadoop.util.Progressable progressable)
boolean	delete(org.apache.hadoop.fs.Path path, boolean b)
boolean	exists(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus	getFileStatus(org.apache.hadoop.fs.Path inlinePath)
String	getScheme()
URI	getUri()
org.apache.hadoop.fs.Path	getWorkingDirectory()
void	initialize(URI name, org.apache.hadoop.conf.Configuration conf)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path inlinePath)
boolean	mkdirs(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission fsPermission)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path inlinePath, int bufferSize)
boolean	rename(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.Path path1)
void	setWorkingDirectory(org.apache.hadoop.fs.Path path)

Methods inherited from class org.apache.hadoop.fs.FileSystem

access, addDelegationTokens, append, append, appendFile, areSymlinksEnabled, cancelDeleteOnExit, canonicalizeUri, checkPath, clearStatistics, close, closeAll, closeAllForUGI, completeLocalOutput, concat, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyToLocalFile, copyToLocalFile, copyToLocalFile, create, create, create, create, create, create, create, create, create, create, create, create, createFile, createNewFile, createNonRecursive, createNonRecursive, createNonRecursive, createSnapshot, createSnapshot, createSymlink, delete, deleteOnExit, deleteSnapshot, enableSymlinks, fixRelativePart, get, get, get, getAclStatus, getAllStatistics, getAllStoragePolicies, getBlockSize, getCanonicalServiceName, getCanonicalUri, getChildFileSystems, getContentSummary, getDefaultBlockSize, getDefaultBlockSize, getDefaultPort, getDefaultReplication, getDefaultReplication, getDefaultUri, getDelegationToken, getFileBlockLocations, getFileBlockLocations, getFileChecksum, getFileChecksum, getFileLinkStatus, getFileSystemClass, getFSofPath, getGlobalStorageStatistics, getHomeDirectory, getInitialWorkingDirectory, getLength, getLinkTarget, getLocal, getName, getNamed, getQuotaUsage, getReplication, getServerDefaults, getServerDefaults, getStatistics, getStatistics, getStatus, getStatus, getStoragePolicy, getStorageStatistics, getTrashRoot, getTrashRoots, getUsed, getUsed, getXAttr, getXAttrs, getXAttrs, globStatus, globStatus, isDirectory, isFile, listCorruptFileBlocks, listFiles, listLocatedStatus, listLocatedStatus, listStatus, listStatus, listStatus, listStatusBatch, listStatusIterator, listXAttrs, makeQualified, mkdirs, mkdirs, modifyAclEntries, moveFromLocalFile, moveFromLocalFile, moveToLocalFile, newInstance, newInstance, newInstance, newInstanceLocal, open, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, removeAcl, removeAclEntries, removeDefaultAcl, removeXAttr, rename, renameSnapshot, resolveLink, resolvePath, setAcl, setDefaultUri, setDefaultUri, setOwner, setPermission, setReplication, setStoragePolicy, setTimes, setVerifyChecksum, setWriteChecksum, setXAttr, setXAttr, startLocalOutput, supportsSymlinks, truncate, unsetStoragePolicy

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

SCHEME

public static final String SCHEME

See Also:

Constant Field Values

Constructor Detail

InLineFileSystem

public InLineFileSystem()

Method Detail

initialize

public void initialize(URI name,
                       org.apache.hadoop.conf.Configuration conf)
                throws IOException

Overrides:

initialize in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getUri

public URI getUri()

Specified by:

getUri in class org.apache.hadoop.fs.FileSystem

getScheme

public String getScheme()

Overrides:

getScheme in class org.apache.hadoop.fs.FileSystem

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path inlinePath,
                                                   int bufferSize)
                                            throws IOException

Specified by:

open in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

exists

public boolean exists(org.apache.hadoop.fs.Path f)

Overrides:

exists in class org.apache.hadoop.fs.FileSystem

getFileStatus

public org.apache.hadoop.fs.FileStatus getFileStatus(org.apache.hadoop.fs.Path inlinePath)
                                              throws IOException

Specified by:

getFileStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path path,
                                                      org.apache.hadoop.fs.permission.FsPermission fsPermission,
                                                      boolean b,
                                                      int i,
                                                      short i1,
                                                      long l,
                                                      org.apache.hadoop.util.Progressable progressable)
                                               throws IOException

Specified by:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path path,
                                                      int i,
                                                      org.apache.hadoop.util.Progressable progressable)
                                               throws IOException

Specified by:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

rename

public boolean rename(org.apache.hadoop.fs.Path path,
                      org.apache.hadoop.fs.Path path1)
               throws IOException

Specified by:

rename in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

delete

public boolean delete(org.apache.hadoop.fs.Path path,
                      boolean b)
               throws IOException

Specified by:

delete in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path inlinePath)
                                             throws IOException

Specified by:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setWorkingDirectory

public void setWorkingDirectory(org.apache.hadoop.fs.Path path)

Specified by:

setWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getWorkingDirectory

public org.apache.hadoop.fs.Path getWorkingDirectory()

Specified by:

getWorkingDirectory in class org.apache.hadoop.fs.FileSystem

mkdirs

public boolean mkdirs(org.apache.hadoop.fs.Path path,
                      org.apache.hadoop.fs.permission.FsPermission fsPermission)
               throws IOException

Specified by:

mkdirs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException