alluxio.underfs.hdfs

Class HdfsUnderFileSystem

java.lang.Object

alluxio.underfs.BaseUnderFileSystem

alluxio.underfs.ConsistentUnderFileSystem

alluxio.underfs.hdfs.HdfsUnderFileSystem

All Implemented Interfaces:

AsyncUfsClient, AtomicFileOutputStreamCallback, UnderFileSystem, java.io.Closeable, java.lang.AutoCloseable

@ThreadSafe
public class HdfsUnderFileSystem
extends ConsistentUnderFileSystem
implements AtomicFileOutputStreamCallback

HDFS UnderFileSystem implementation.

Nested Class Summary

Nested classes/interfaces inherited from interface alluxio.underfs.UnderFileSystem

UnderFileSystem.Factory, UnderFileSystem.SpaceType

Field Summary

Fields

Modifier and Type	Field and Description
protected static java.lang.String	CHECKSUM_COMBINE_MODE
protected static java.lang.String	HADOOP_AUTH_METHOD
protected static java.lang.String	HDFS_ACL_PROVIDER_CLASS Name of the class for the HDFS Acl provider.
protected static java.lang.String	HDFS_EC_CODEC_REGISTRY_CLASS Name of the class for the HDFS EC Codec Registry.
protected static java.lang.String	HDFS_EC_MIN_VERSION The minimum HDFS production version required for EC.
protected static java.lang.String	HDFS_USER
protected static java.lang.String	JAVAX_WS_RS_CORE_MEDIA_TYPE
protected static java.lang.String	KRB_KEYTAB_LOGIN_AUTO_RENEW
protected static java.lang.String	KRB5_CONF_FILE
protected static int	MAX_TRY
protected HdfsAclProvider	mHdfsAclProvider
protected static java.lang.String	USER_NAMESPACE_PREFIX

Fields inherited from class alluxio.underfs.BaseUnderFileSystem

EMPTY_ACL, mUfsConf, mUri

Constructor Summary

Constructors

Modifier	Constructor and Description
	HdfsUnderFileSystem(AlluxioURI ufsUri, UnderFileSystemConfiguration conf, org.apache.hadoop.conf.Configuration hdfsConf) Constructs a new HDFS UnderFileSystem.
protected	HdfsUnderFileSystem(AlluxioURI ufsUri, UnderFileSystemConfiguration conf, org.apache.hadoop.conf.Configuration hdfsConf, boolean useLoadingCache)

Method Summary

Modifier and Type	Method and Description
void	cleanup()
void	close()
void	connectFromMaster(java.lang.String host)
void	connectFromWorker(java.lang.String host)
java.io.OutputStream	create(java.lang.String path, CreateOptions options)
static org.apache.hadoop.conf.Configuration	createConfiguration(UnderFileSystemConfiguration conf) Prepares the Hadoop configuration necessary to successfully obtain a FileSystem instance that can access the provided path.
java.io.OutputStream	createDirect(java.lang.String path, CreateOptions options)
static HdfsUnderFileSystem	createInstance(AlluxioURI ufsUri, UnderFileSystemConfiguration conf) Factory method to constructs a new HDFS UnderFileSystem instance.
boolean	deleteDirectory(java.lang.String path, DeleteOptions options)
boolean	deleteFile(java.lang.String path)
boolean	exists(java.lang.String path)
Pair<AccessControlList,DefaultAccessControlList>	getAclPair(java.lang.String path)
java.util.Map<java.lang.String,java.lang.String>	getAttributes(java.lang.String path)
long	getBlockSizeByte(java.lang.String path)
UfsDirectoryStatus	getDirectoryStatus(java.lang.String path)
java.util.List<java.lang.String>	getFileLocations(java.lang.String path)
java.util.List<java.lang.String>	getFileLocations(java.lang.String path, FileLocationOptions options)
UfsFileStatus	getFileStatus(java.lang.String path, GetStatusOptions options)
org.apache.hadoop.fs.FileSystem	getFs()
long	getSpace(java.lang.String path, UnderFileSystem.SpaceType type)
UfsStatus	getStatus(java.lang.String path, GetStatusOptions options)
java.lang.String	getUnderFSType()
boolean	isDirectory(java.lang.String path)
boolean	isFile(java.lang.String path)
boolean	isSeekable()
UfsStatus[]	listStatus(java.lang.String path)
java.util.Iterator<UfsStatus>	listStatusIterable(java.lang.String path, ListOptions options, java.lang.String startAfter, int batchSize)
boolean	mkdirs(java.lang.String path, MkdirsOptions options)
java.io.InputStream	open(java.lang.String path, OpenOptions options)
HdfsPositionedUnderFileInputStream	openPositionRead(java.lang.String path, long fileLength)
boolean	renameDirectory(java.lang.String src, java.lang.String dst)
boolean	renameFile(java.lang.String src, java.lang.String dst)
void	setAclEntries(java.lang.String path, java.util.List<AclEntry> aclEntries)
void	setAttribute(java.lang.String path, java.lang.String name, byte[] value)
void	setMode(java.lang.String path, short mode)
void	setOwner(java.lang.String path, java.lang.String user, java.lang.String group)
boolean	supportsFlush()

Methods inherited from class alluxio.underfs.ConsistentUnderFileSystem

createNonexistingFile, createNonexistingFile, deleteExistingDirectory, deleteExistingDirectory, deleteExistingFile, getExistingDirectoryStatus, getExistingFileStatus, getExistingStatus, isExistingDirectory, openExistingFile, openExistingFile, renameRenamableDirectory, renameRenamableFile

Methods inherited from class alluxio.underfs.BaseUnderFileSystem

create, deleteDirectory, getConfiguration, getFingerprint, getOperationMode, getParsedFingerprint, getParsedFingerprint, getPhysicalStores, isObjectStorage, listStatus, mkdirs, open, performListingAsync, resolveUri, validatePath

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface alluxio.underfs.UnderFileSystem

create, createNonexistingFile, createNonexistingFile, deleteDirectory, deleteExistingDirectory, deleteExistingDirectory, deleteExistingFile, getConfiguration, getExistingDirectoryStatus, getExistingFileStatus, getExistingStatus, getFileStatus, getFingerprint, getOperationMode, getParsedFingerprint, getParsedFingerprint, getPhysicalStores, getStatus, getTemporaryToken, isExistingDirectory, isObjectStorage, isTemporaryConnectionToken, listStatus, listStatuses, mkdirs, open, openExistingFile, openExistingFile, refreshTemporaryToken, renameRenamableDirectory, renameRenamableFile, resolveUri, setTemporaryTokenRefreshContext

Methods inherited from interface alluxio.underfs.AsyncUfsClient

performListingAsync

Field Detail

MAX_TRY

protected static final int MAX_TRY

See Also:

Constant Field Values

HDFS_USER

protected static final java.lang.String HDFS_USER

See Also:

Constant Field Values

HDFS_ACL_PROVIDER_CLASS

protected static final java.lang.String HDFS_ACL_PROVIDER_CLASS

Name of the class for the HDFS Acl provider.

See Also:

Constant Field Values

HDFS_EC_MIN_VERSION

protected static final java.lang.String HDFS_EC_MIN_VERSION

The minimum HDFS production version required for EC.

See Also:

Constant Field Values

HDFS_EC_CODEC_REGISTRY_CLASS

protected static final java.lang.String HDFS_EC_CODEC_REGISTRY_CLASS

Name of the class for the HDFS EC Codec Registry.

See Also:

Constant Field Values

JAVAX_WS_RS_CORE_MEDIA_TYPE

protected static final java.lang.String JAVAX_WS_RS_CORE_MEDIA_TYPE

See Also:

Constant Field Values

HADOOP_AUTH_METHOD

protected static final java.lang.String HADOOP_AUTH_METHOD

See Also:

Constant Field Values

KRB5_CONF_FILE

protected static final java.lang.String KRB5_CONF_FILE

See Also:

Constant Field Values

KRB_KEYTAB_LOGIN_AUTO_RENEW

protected static final java.lang.String KRB_KEYTAB_LOGIN_AUTO_RENEW

See Also:

Constant Field Values

CHECKSUM_COMBINE_MODE

protected static final java.lang.String CHECKSUM_COMBINE_MODE

See Also:

Constant Field Values

USER_NAMESPACE_PREFIX

protected static final java.lang.String USER_NAMESPACE_PREFIX

See Also:

Constant Field Values

mHdfsAclProvider

protected final HdfsAclProvider mHdfsAclProvider

Constructor Detail

HdfsUnderFileSystem

public HdfsUnderFileSystem(AlluxioURI ufsUri,
                           UnderFileSystemConfiguration conf,
                           org.apache.hadoop.conf.Configuration hdfsConf)

Constructs a new HDFS UnderFileSystem.

Parameters:

ufsUri - the AlluxioURI for this UFS

conf - the configuration for this UFS

hdfsConf - the configuration for HDFS

HdfsUnderFileSystem

protected HdfsUnderFileSystem(AlluxioURI ufsUri,
                              UnderFileSystemConfiguration conf,
                              org.apache.hadoop.conf.Configuration hdfsConf,
                              boolean useLoadingCache)

Method Detail

createInstance

public static HdfsUnderFileSystem createInstance(AlluxioURI ufsUri,
                                                 UnderFileSystemConfiguration conf)

Factory method to constructs a new HDFS UnderFileSystem instance.

Parameters:

ufsUri - the AlluxioURI for this UFS

conf - the configuration for Hadoop

Returns:

a new HDFS UnderFileSystem instance

getUnderFSType

public java.lang.String getUnderFSType()

Specified by:

getUnderFSType in interface UnderFileSystem

createConfiguration

public static org.apache.hadoop.conf.Configuration createConfiguration(UnderFileSystemConfiguration conf)

Prepares the Hadoop configuration necessary to successfully obtain a FileSystem instance that can access the provided path.

Derived implementations that work with specialised Hadoop FileSystem API compatible implementations can override this method to add implementation specific configuration necessary for obtaining a usable FileSystem instance.

Parameters:

conf - the configuration for this UFS

Returns:

the configuration for HDFS

cleanup

public void cleanup()

Specified by:

cleanup in interface UnderFileSystem

close

public void close()

Specified by:

close in interface java.io.Closeable

Specified by:

close in interface java.lang.AutoCloseable

Overrides:

close in class BaseUnderFileSystem

create

public java.io.OutputStream create(java.lang.String path,
                                   CreateOptions options)
                            throws java.io.IOException

Specified by:

create in interface UnderFileSystem

Throws:

java.io.IOException

createDirect

public java.io.OutputStream createDirect(java.lang.String path,
                                         CreateOptions options)
                                  throws java.io.IOException

Specified by:

createDirect in interface AtomicFileOutputStreamCallback

Throws:

java.io.IOException

deleteDirectory

public boolean deleteDirectory(java.lang.String path,
                               DeleteOptions options)
                        throws java.io.IOException

Specified by:

deleteDirectory in interface UnderFileSystem

Throws:

java.io.IOException

deleteFile

public boolean deleteFile(java.lang.String path)
                   throws java.io.IOException

Specified by:

deleteFile in interface UnderFileSystem

Throws:

java.io.IOException

exists

public boolean exists(java.lang.String path)
               throws java.io.IOException

Specified by:

exists in interface UnderFileSystem

Overrides:

exists in class BaseUnderFileSystem

Throws:

java.io.IOException

getAclPair

public Pair<AccessControlList,DefaultAccessControlList> getAclPair(java.lang.String path)
                                                            throws java.io.IOException

Specified by:

getAclPair in interface UnderFileSystem

Overrides:

getAclPair in class BaseUnderFileSystem

Throws:

java.io.IOException

setAclEntries

public void setAclEntries(java.lang.String path,
                          java.util.List<AclEntry> aclEntries)
                   throws java.io.IOException

Specified by:

setAclEntries in interface UnderFileSystem

Overrides:

setAclEntries in class BaseUnderFileSystem

Throws:

java.io.IOException

getBlockSizeByte

public long getBlockSizeByte(java.lang.String path)
                      throws java.io.IOException

Specified by:

getBlockSizeByte in interface UnderFileSystem

Throws:

java.io.IOException

getDirectoryStatus

public UfsDirectoryStatus getDirectoryStatus(java.lang.String path)
                                      throws java.io.IOException

Specified by:

getDirectoryStatus in interface UnderFileSystem

Throws:

java.io.IOException

getFileLocations

public java.util.List<java.lang.String> getFileLocations(java.lang.String path)
                                                  throws java.io.IOException

Specified by:

getFileLocations in interface UnderFileSystem

Throws:

java.io.IOException

getFileLocations

@Nullable
public java.util.List<java.lang.String> getFileLocations(java.lang.String path,
                                                                   FileLocationOptions options)
                                                            throws java.io.IOException

Specified by:

getFileLocations in interface UnderFileSystem

Throws:

java.io.IOException

getFileStatus

public UfsFileStatus getFileStatus(java.lang.String path,
                                   GetStatusOptions options)
                            throws java.io.IOException

Specified by:

getFileStatus in interface UnderFileSystem

Throws:

java.io.IOException

getSpace

public long getSpace(java.lang.String path,
                     UnderFileSystem.SpaceType type)
              throws java.io.IOException

Specified by:

getSpace in interface UnderFileSystem

Throws:

java.io.IOException

getStatus

public UfsStatus getStatus(java.lang.String path,
                           GetStatusOptions options)
                    throws java.io.IOException

Specified by:

getStatus in interface UnderFileSystem

Throws:

java.io.IOException

isDirectory

public boolean isDirectory(java.lang.String path)
                    throws java.io.IOException

Specified by:

isDirectory in interface UnderFileSystem

Throws:

java.io.IOException

isFile

public boolean isFile(java.lang.String path)
               throws java.io.IOException

Specified by:

isFile in interface UnderFileSystem

Throws:

java.io.IOException

listStatusIterable

@Nullable
public java.util.Iterator<UfsStatus> listStatusIterable(java.lang.String path,
                                                                  ListOptions options,
                                                                  java.lang.String startAfter,
                                                                  int batchSize)
                                                           throws java.io.IOException

Specified by:

listStatusIterable in interface UnderFileSystem

Overrides:

listStatusIterable in class BaseUnderFileSystem

Throws:

java.io.IOException

listStatus

@Nullable
public UfsStatus[] listStatus(java.lang.String path)
                                 throws java.io.IOException

Specified by:

listStatus in interface UnderFileSystem

Throws:

java.io.IOException

connectFromMaster

public void connectFromMaster(java.lang.String host)
                       throws java.io.IOException

Specified by:

connectFromMaster in interface UnderFileSystem

Throws:

java.io.IOException

connectFromWorker

public void connectFromWorker(java.lang.String host)
                       throws java.io.IOException

Specified by:

connectFromWorker in interface UnderFileSystem

Throws:

java.io.IOException

mkdirs

public boolean mkdirs(java.lang.String path,
                      MkdirsOptions options)
               throws java.io.IOException

Specified by:

mkdirs in interface UnderFileSystem

Throws:

java.io.IOException

open

public java.io.InputStream open(java.lang.String path,
                                OpenOptions options)
                         throws java.io.IOException

Specified by:

open in interface UnderFileSystem

Throws:

java.io.IOException

openPositionRead

public HdfsPositionedUnderFileInputStream openPositionRead(java.lang.String path,
                                                           long fileLength)

Specified by:

openPositionRead in interface UnderFileSystem

renameDirectory

public boolean renameDirectory(java.lang.String src,
                               java.lang.String dst)
                        throws java.io.IOException

Specified by:

renameDirectory in interface UnderFileSystem

Throws:

java.io.IOException

renameFile

public boolean renameFile(java.lang.String src,
                          java.lang.String dst)
                   throws java.io.IOException

Specified by:

renameFile in interface UnderFileSystem

Throws:

java.io.IOException

setOwner

public void setOwner(java.lang.String path,
                     java.lang.String user,
                     java.lang.String group)
              throws java.io.IOException

Specified by:

setOwner in interface UnderFileSystem

Throws:

java.io.IOException

setAttribute

public void setAttribute(java.lang.String path,
                         java.lang.String name,
                         byte[] value)
                  throws java.io.IOException

Specified by:

setAttribute in interface UnderFileSystem

Throws:

java.io.IOException

getAttributes

public java.util.Map<java.lang.String,java.lang.String> getAttributes(java.lang.String path)
                                                               throws java.io.IOException

Specified by:

getAttributes in interface UnderFileSystem

Throws:

java.io.IOException

setMode

public void setMode(java.lang.String path,
                    short mode)
             throws java.io.IOException

Specified by:

setMode in interface UnderFileSystem

Throws:

java.io.IOException

supportsFlush

public boolean supportsFlush()

Specified by:

supportsFlush in interface UnderFileSystem

isSeekable

public boolean isSeekable()

Specified by:

isSeekable in interface UnderFileSystem

Overrides:

isSeekable in class BaseUnderFileSystem

getFs

public org.apache.hadoop.fs.FileSystem getFs()
                                      throws java.io.IOException

Returns:

the underlying HDFS FileSystem object

Throws:

java.io.IOException