com.google.cloud.hadoop.fs.gcs

Class GoogleHadoopFS

java.lang.Object

AbstractFileSystem

com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS

public class GoogleHadoopFS
extends AbstractFileSystem

GoogleHadoopFS provides a YARN compatible Abstract File System on top of Google Cloud Storage (GCS). It is implemented as a thin abstraction layer on top of GoogleHadoopFileSystem, but will soon be refactored to share a common base.

Constructor Summary

Constructors

Constructor and Description
GoogleHadoopFS(GoogleHadoopFileSystem ghfs, java.net.URI uri, org.apache.hadoop.conf.Configuration conf)
GoogleHadoopFS(java.net.URI uri, org.apache.hadoop.conf.Configuration conf)

Method Summary

Modifier and Type	Method and Description
void	checkPath(org.apache.hadoop.fs.Path path) Only accept valid AbstractFileSystem and GoogleHadoopFileSystem Paths.
org.apache.hadoop.fs.FSDataOutputStream	createInternal(org.apache.hadoop.fs.Path file, java.util.EnumSet<CreateFlag> flag, org.apache.hadoop.fs.permission.FsPermission absolutePermission, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress, ChecksumOpt checksumOpt, boolean createParent)
boolean	delete(org.apache.hadoop.fs.Path f, boolean recursive)
org.apache.hadoop.fs.BlockLocation[]	getFileBlockLocations(org.apache.hadoop.fs.Path f, long start, long len)
org.apache.hadoop.fs.FileChecksum	getFileChecksum(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus	getFileStatus(org.apache.hadoop.fs.Path f)
FsStatus	getFsStatus()
FsServerDefaults	getServerDefaults()
java.net.URI	getUri() This is overridden to use GoogleHadoopFileSystem's URI, because AbstractFileSystem appends the default port to the authority.
int	getUriDefaultPort()
boolean	isValidName(java.lang.String src) Follow HDFS conventions except allow for ':' in paths.
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path f)
void	mkdir(org.apache.hadoop.fs.Path dir, org.apache.hadoop.fs.permission.FsPermission permission, boolean createParent)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path f, int bufferSize)
void	renameInternal(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	setOwner(org.apache.hadoop.fs.Path f, java.lang.String username, java.lang.String groupname)
void	setPermission(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission)
boolean	setReplication(org.apache.hadoop.fs.Path f, short replication)
void	setTimes(org.apache.hadoop.fs.Path f, long mtime, long atime)
void	setVerifyChecksum(boolean verifyChecksum)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

GoogleHadoopFS

public GoogleHadoopFS(java.net.URI uri,
                      org.apache.hadoop.conf.Configuration conf)
               throws java.net.URISyntaxException,
                      java.io.IOException

Throws:

java.net.URISyntaxException

java.io.IOException

GoogleHadoopFS

public GoogleHadoopFS(GoogleHadoopFileSystem ghfs,
                      java.net.URI uri,
                      org.apache.hadoop.conf.Configuration conf)
               throws java.net.URISyntaxException,
                      java.io.IOException

Throws:

java.net.URISyntaxException

java.io.IOException

Method Detail

createInternal

public org.apache.hadoop.fs.FSDataOutputStream createInternal(org.apache.hadoop.fs.Path file,
                                                              java.util.EnumSet<CreateFlag> flag,
                                                              org.apache.hadoop.fs.permission.FsPermission absolutePermission,
                                                              int bufferSize,
                                                              short replication,
                                                              long blockSize,
                                                              org.apache.hadoop.util.Progressable progress,
                                                              ChecksumOpt checksumOpt,
                                                              boolean createParent)
                                                       throws java.io.IOException

Throws:

java.io.IOException

getUriDefaultPort

public int getUriDefaultPort()

getUri

public java.net.URI getUri()

This is overridden to use GoogleHadoopFileSystem's URI, because AbstractFileSystem appends the default port to the authority.

isValidName

public boolean isValidName(java.lang.String src)

Follow HDFS conventions except allow for ':' in paths.

checkPath

public void checkPath(org.apache.hadoop.fs.Path path)

Only accept valid AbstractFileSystem and GoogleHadoopFileSystem Paths.

getServerDefaults

public FsServerDefaults getServerDefaults()
                                   throws java.io.IOException

Throws:

java.io.IOException

mkdir

public void mkdir(org.apache.hadoop.fs.Path dir,
                  org.apache.hadoop.fs.permission.FsPermission permission,
                  boolean createParent)
           throws java.io.IOException

Throws:

java.io.IOException

delete

public boolean delete(org.apache.hadoop.fs.Path f,
                      boolean recursive)
               throws java.io.IOException

Throws:

java.io.IOException

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f,
                                                   int bufferSize)
                                            throws java.io.IOException

Throws:

java.io.IOException

setReplication

public boolean setReplication(org.apache.hadoop.fs.Path f,
                              short replication)
                       throws java.io.IOException

Throws:

java.io.IOException

renameInternal

public void renameInternal(org.apache.hadoop.fs.Path src,
                           org.apache.hadoop.fs.Path dst)
                    throws java.io.IOException

Throws:

java.io.IOException

setPermission

public void setPermission(org.apache.hadoop.fs.Path f,
                          org.apache.hadoop.fs.permission.FsPermission permission)
                   throws java.io.IOException

Throws:

java.io.IOException

setOwner

public void setOwner(org.apache.hadoop.fs.Path f,
                     java.lang.String username,
                     java.lang.String groupname)
              throws java.io.IOException

Throws:

java.io.IOException

setTimes

public void setTimes(org.apache.hadoop.fs.Path f,
                     long mtime,
                     long atime)
              throws java.io.IOException

Throws:

java.io.IOException

getFileChecksum

public org.apache.hadoop.fs.FileChecksum getFileChecksum(org.apache.hadoop.fs.Path f)
                                                  throws java.io.IOException

Throws:

java.io.IOException

getFileStatus

public org.apache.hadoop.fs.FileStatus getFileStatus(org.apache.hadoop.fs.Path f)
                                              throws java.io.IOException

Throws:

java.io.IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.Path f,
                                                                  long start,
                                                                  long len)
                                                           throws java.io.IOException

Throws:

java.io.IOException

getFsStatus

public FsStatus getFsStatus()
                     throws java.io.IOException

Throws:

java.io.IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path f)
                                             throws java.io.IOException

Throws:

java.io.IOException

setVerifyChecksum

public void setVerifyChecksum(boolean verifyChecksum)