water.persist

Class PersistHdfs

java.lang.Object

water.persist.Persist

water.persist.PersistHdfs

public final class PersistHdfs
extends water.persist.Persist

HDFS persistence layer.

Nested Class Summary

Nested classes/interfaces inherited from class water.persist.Persist

water.persist.Persist.PersistEntry

Field Summary

Fields

Modifier and Type	Field and Description
static org.apache.hadoop.conf.Configuration	CONF Globally shared HDFS configuration.

Constructor Summary

Constructors

Constructor and Description
PersistHdfs()
PersistHdfs(java.net.URI uri)

Method Summary

Modifier and Type	Method and Description
static void	addFolder(org.apache.hadoop.fs.Path p, java.util.ArrayList<java.lang.String> keys, java.util.ArrayList<java.lang.String> failed)
java.util.ArrayList<java.lang.String>	calcTypeaheadMatches(java.lang.String filter, int limit)
boolean	canHandle(java.lang.String path)
void	cleanUp()
java.io.OutputStream	create(java.lang.String path, boolean overwrite)
boolean	delete(java.lang.String path)
void	delete(water.Value v)
boolean	exists(java.lang.String path)
static org.apache.hadoop.fs.FileSystem	getFS(java.lang.String path)
static org.apache.hadoop.fs.FileSystem	getFS(java.net.URI uri)
java.lang.String	getHomeDirectory()
java.lang.String	getParent(java.lang.String path)
void	importFiles(java.lang.String path, java.lang.String pattern, java.util.ArrayList<java.lang.String> files, java.util.ArrayList<java.lang.String> keys, java.util.ArrayList<java.lang.String> fails, java.util.ArrayList<java.lang.String> dels)
boolean	isDirectory(java.lang.String path)
boolean	isSeekableOpenSupported()
long	length(java.lang.String path)
water.persist.Persist.PersistEntry[]	list(java.lang.String path)
byte[]	load(water.Key k, long skip, int max)
byte[]	load(water.Value v) InputStream from a HDFS-based Key
boolean	mkdirs(java.lang.String path)
java.io.InputStream	open(java.lang.String path)
java.io.InputStream	openSeekable(java.lang.String path)
boolean	rename(java.lang.String fromPath, java.lang.String toPath)
static void	store(org.apache.hadoop.fs.Path path, byte[] data)
void	store(water.Value v)
water.Key	uriToKey(java.net.URI uri)
java.io.InputStream	wrapSeekable(water.fvec.Vec vec)

Methods inherited from class water.persist.Persist

getTotalSpace, getUsableSpace, key2Str

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CONF

public static final org.apache.hadoop.conf.Configuration CONF

Globally shared HDFS configuration.

Constructor Detail

PersistHdfs

public PersistHdfs()

PersistHdfs

public PersistHdfs(java.net.URI uri)

Method Detail

cleanUp

public void cleanUp()

Specified by:

cleanUp in class water.persist.Persist

load

public byte[] load(water.Value v)

InputStream from a HDFS-based Key

Specified by:

load in class water.persist.Persist

load

public byte[] load(water.Key k,
                   long skip,
                   int max)

Overrides:

load in class water.persist.Persist

store

public void store(water.Value v)

Specified by:

store in class water.persist.Persist

store

public static void store(org.apache.hadoop.fs.Path path,
                         byte[] data)

delete

public void delete(water.Value v)

Specified by:

delete in class water.persist.Persist

addFolder

public static void addFolder(org.apache.hadoop.fs.Path p,
                             java.util.ArrayList<java.lang.String> keys,
                             java.util.ArrayList<java.lang.String> failed)
                      throws java.io.IOException

Throws:

java.io.IOException

uriToKey

public water.Key uriToKey(java.net.URI uri)
                   throws java.io.IOException

Specified by:

uriToKey in class water.persist.Persist

Throws:

java.io.IOException

getFS

public static org.apache.hadoop.fs.FileSystem getFS(java.lang.String path)
                                             throws java.io.IOException

Throws:

java.io.IOException

getFS

public static org.apache.hadoop.fs.FileSystem getFS(java.net.URI uri)
                                             throws java.io.IOException

Throws:

java.io.IOException

calcTypeaheadMatches

public java.util.ArrayList<java.lang.String> calcTypeaheadMatches(java.lang.String filter,
                                                                  int limit)

Specified by:

calcTypeaheadMatches in class water.persist.Persist

importFiles

public void importFiles(java.lang.String path,
                        java.lang.String pattern,
                        java.util.ArrayList<java.lang.String> files,
                        java.util.ArrayList<java.lang.String> keys,
                        java.util.ArrayList<java.lang.String> fails,
                        java.util.ArrayList<java.lang.String> dels)

Specified by:

importFiles in class water.persist.Persist

getHomeDirectory

public java.lang.String getHomeDirectory()

Overrides:

getHomeDirectory in class water.persist.Persist

list

public water.persist.Persist.PersistEntry[] list(java.lang.String path)

Overrides:

list in class water.persist.Persist

exists

public boolean exists(java.lang.String path)

Overrides:

exists in class water.persist.Persist

getParent

public java.lang.String getParent(java.lang.String path)

Overrides:

getParent in class water.persist.Persist

isDirectory

public boolean isDirectory(java.lang.String path)

Overrides:

isDirectory in class water.persist.Persist

length

public long length(java.lang.String path)

Overrides:

length in class water.persist.Persist

open

public java.io.InputStream open(java.lang.String path)

Overrides:

open in class water.persist.Persist

openSeekable

public java.io.InputStream openSeekable(java.lang.String path)

Overrides:

openSeekable in class water.persist.Persist

wrapSeekable

public java.io.InputStream wrapSeekable(water.fvec.Vec vec)

Overrides:

wrapSeekable in class water.persist.Persist

isSeekableOpenSupported

public boolean isSeekableOpenSupported()

Overrides:

isSeekableOpenSupported in class water.persist.Persist

mkdirs

public boolean mkdirs(java.lang.String path)

Overrides:

mkdirs in class water.persist.Persist

rename

public boolean rename(java.lang.String fromPath,
                      java.lang.String toPath)

Overrides:

rename in class water.persist.Persist

create

public java.io.OutputStream create(java.lang.String path,
                                   boolean overwrite)

Overrides:

create in class water.persist.Persist

delete

public boolean delete(java.lang.String path)

Overrides:

delete in class water.persist.Persist

canHandle

public boolean canHandle(java.lang.String path)

Overrides:

canHandle in class water.persist.Persist