org.apache.hadoop.hive.ql.io

Class HdfsUtils

java.lang.Object

org.apache.hadoop.hive.ql.io.HdfsUtils

public class HdfsUtils
extends Object

Common FileSystem utilities around FileId.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	HdfsUtils.HdfsFileStatusWithoutId

Constructor Summary

Constructors

Constructor and Description
HdfsUtils()

Method Summary

Modifier and Type	Method and Description
static long	createTestFileId(String pathStr, org.apache.hadoop.fs.FileStatus fs, boolean doLog, String fsName)
static Object	getFileId(org.apache.hadoop.fs.FileSystem fileSystem, org.apache.hadoop.fs.Path path, boolean allowSynthetic, boolean checkDefaultFs, boolean forceSyntheticIds)
static org.apache.hadoop.fs.Path	getFileIdPath(org.apache.hadoop.fs.Path path, long fileId)
static boolean	isDefaultFs(org.apache.hadoop.hdfs.DistributedFileSystem fs)
static List<org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId>	listFileStatusWithId(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir, Ref<Boolean> useFileIds, boolean recursive, org.apache.hadoop.fs.PathFilter filter) List files recursively in the given directory.
static List<org.apache.hadoop.fs.FileStatus>	listLocatedFileStatus(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.PathFilter filter, boolean recursive) List filestatuses in the given directory.
static List<org.apache.hadoop.fs.Path>	listPath(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.PathFilter filter, boolean recursive)
static boolean	pathExists(org.apache.hadoop.fs.Path p, org.apache.hadoop.conf.Configuration conf)
static List<org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId>	tryListLocatedHdfsStatus(Ref<Boolean> useFileIds, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path directory, org.apache.hadoop.fs.PathFilter filter)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HdfsUtils

public HdfsUtils()

Method Detail

getFileId

public static Object getFileId(org.apache.hadoop.fs.FileSystem fileSystem,
                               org.apache.hadoop.fs.Path path,
                               boolean allowSynthetic,
                               boolean checkDefaultFs,
                               boolean forceSyntheticIds)
                        throws IOException

Throws:

IOException

createTestFileId

public static long createTestFileId(String pathStr,
                                    org.apache.hadoop.fs.FileStatus fs,
                                    boolean doLog,
                                    String fsName)

listLocatedFileStatus

public static List<org.apache.hadoop.fs.FileStatus> listLocatedFileStatus(org.apache.hadoop.fs.FileSystem fs,
                                                                          org.apache.hadoop.fs.Path path,
                                                                          org.apache.hadoop.fs.PathFilter filter,
                                                                          boolean recursive)
                                                                   throws IOException

List filestatuses in the given directory. In recursive mode all files are returned, otherwise directories are skipped.

Parameters:

fs - FileSystem

path - directory to list

filter - optional filter

recursive - do recursive listing

Returns:

Throws:

IOException

listPath

public static List<org.apache.hadoop.fs.Path> listPath(org.apache.hadoop.fs.FileSystem fs,
                                                       org.apache.hadoop.fs.Path path,
                                                       org.apache.hadoop.fs.PathFilter filter,
                                                       boolean recursive)
                                                throws IOException

Throws:

IOException

getFileIdPath

public static org.apache.hadoop.fs.Path getFileIdPath(org.apache.hadoop.fs.Path path,
                                                      long fileId)

isDefaultFs

public static boolean isDefaultFs(org.apache.hadoop.hdfs.DistributedFileSystem fs)

pathExists

public static boolean pathExists(org.apache.hadoop.fs.Path p,
                                 org.apache.hadoop.conf.Configuration conf)
                          throws HiveException

Throws:

HiveException

listFileStatusWithId

public static List<org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId> listFileStatusWithId(org.apache.hadoop.fs.FileSystem fs,
                                                                                                       org.apache.hadoop.fs.Path dir,
                                                                                                       Ref<Boolean> useFileIds,
                                                                                                       boolean recursive,
                                                                                                       org.apache.hadoop.fs.PathFilter filter)
                                                                                                throws IOException

List files recursively in the given directory. If the filesystem supports it will use file listing with FileIds.

Parameters:

fs - the filesystem

dir - the directory to list

useFileIds - use fileId if possible

recursive - do recursive listing

filter - filter to apply

Returns:

List of files

Throws:

IOException - ex

tryListLocatedHdfsStatus

public static List<org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId> tryListLocatedHdfsStatus(Ref<Boolean> useFileIds,
                                                                                                           org.apache.hadoop.fs.FileSystem fs,
                                                                                                           org.apache.hadoop.fs.Path directory,
                                                                                                           org.apache.hadoop.fs.PathFilter filter)