com.ksyun.kmr.hadoop.fs.ks3

类 Ks3FileSystemStore

java.lang.Object

com.ksyun.kmr.hadoop.fs.ks3.Ks3FileSystemStore

public class Ks3FileSystemStore
extends Object

嵌套类概要

嵌套类

限定符和类型	类和说明
static interface	Ks3FileSystemStore.EngineSender

字段概要

字段

限定符和类型	字段和说明
int	blockSize
String	bucket
FileSystemStoreContext	context
int	greater_than_this_use_parallel_when_destroy
com.ksyun.ks3.service.Ks3	ks3Client
int	parallel_commit_pool_size
int	parallel_commit_speed_limit
int	parallel_commit_thread_size
int	parallel_copy_part_pool_size
int	parallel_copy_part_speed_limit
int	parallel_copy_part_thread_size
int	parallel_copy_pool_size
int	parallel_copy_speed_limit
int	parallel_copy_thread_size
int	parallel_delete_pool_size
int	parallel_delete_speed_limit
int	parallel_delete_thread_size
int	parallel_unser_pool_size
int	parallel_unser_speed_limit
int	parallel_unser_thread_size
int	parallel_upload_part_limit
int	parallel_upload_part_pool_size
int	parallel_upload_part_thread_size

构造器概要

构造器

构造器和说明
Ks3FileSystemStore()

方法概要

限定符和类型	方法和说明
void	checkIsRequestCompress(com.ksyun.ks3.service.request.Ks3WebServiceRequest request)
void	completeMultipartUpload(String objectKey, String uploadId, List<com.ksyun.ks3.dto.PartETag> eTags)
void	completeMultipartUpload(String objectKey, String uploadId, List<com.ksyun.ks3.dto.PartETag> eTags, com.google.common.util.concurrent.RateLimiter rateLimiter)
void	copyObject(com.ksyun.ks3.dto.ObjectMetadata srcMeta, String srcObjectKey, String dstObjectKey)
void	copyObject(com.ksyun.ks3.dto.ObjectMetadata srcMeta, String srcObjectKey, String dstObjectKey, com.google.common.util.concurrent.RateLimiter rateLimiter)
void	copyObject(String srcObjectKey, String dstObjectKey)
void	copyObjects(List<org.apache.commons.lang3.tuple.Pair<String,String>> keys)
void	copyPart(com.google.common.util.concurrent.RateLimiter rateLimiter, CopyPartBean bean)
void	createEmptyObject(String objectKey)
void	createEmptyObject(String objectKey, com.google.common.util.concurrent.RateLimiter rateLimiter)
void	deleteDir(String key, boolean skip404)
void	deleteObject(String objectKey)
void	deleteObject(String objectKey, com.google.common.util.concurrent.RateLimiter rateLimiter)
void	deleteObjects(List<String> objectKeys)
void	deleteObjectSkip404(String objectKey)
void	deleteObjectSkip404(String objectKey, com.google.common.util.concurrent.RateLimiter rateLimiter)
int	getBlockSize()
static org.apache.commons.lang3.tuple.Pair<String,String>	getBucketAndEndpoint(URI uri, String defaultBucket)
org.apache.hadoop.conf.Configuration	getConf()
int	getCopyBlockSize()
com.ksyun.ks3.dto.ObjectMetadata	getMetadata(String objectKey)
com.ksyun.ks3.dto.ObjectMetadata	getMetadata(String objectKey, com.google.common.util.concurrent.RateLimiter rateLimiter)
com.ksyun.ks3.AutoAbortInputStream	getObject(String objectKey)
com.ksyun.ks3.AutoAbortInputStream	getObject(String objectKey, long contentLength, long pos)
com.ksyun.ks3.AutoAbortInputStream	getObject(String objectKey, com.google.common.util.concurrent.RateLimiter rateLimiter)
long	getOverallCopyMaxLen()
org.apache.hadoop.fs.FileSystem.Statistics	getStatistics()
void	initContext()
void	initialize(URI uri, org.apache.hadoop.fs.Path workingDir, org.apache.hadoop.fs.FileSystem.Statistics statistics, org.apache.hadoop.conf.Configuration conf)
String	initMultipartUpload(String objectKey)
String	initMultipartUpload(String objectKey, com.google.common.util.concurrent.RateLimiter rateLimiter)
void	initParallelConfig()
ListObjectsResult	listAllObjects(String objectKey)
ListObjectsResult	listAllObjects(String objectKey, int limit)
ListObjectsResult	listAllObjects(String objectKey, int limit, int pageNum)
ListObjectsResult	listAllSubPaths(String objectKey)
ListObjectsResult	listAllSubPaths(String objectKey, int limit)
ListObjectsResult	listAllSubPaths(String objectKey, int limit, int pageNum)
void	parallelDeleteObjects(Ks3FileSystemStore.EngineSender runnable)
void	parallelDeleteObjects(Ks3FileSystemStore.EngineSender runnable, int rateLimit)
void	parallelDeleteObjects(List<String> objectKeys)
String	pathToKey(org.apache.hadoop.fs.Path path)
void	putObject(Ks3BlockBuffer ks3BlockBuffer, int blockLength)
void	putObject(String objectKey, byte[] bytes)
void	putObject(String objectKey, File file)
void	renewKs3Client()
void	serialDeleteObjects(List<String> objectKeys)
com.ksyun.ks3.dto.PartETag	uploadPart(Ks3BlockBuffer ks3BlockBuffer, String uploadId, com.google.common.util.concurrent.RateLimiter rateLimiter, int blockLength)
com.ksyun.ks3.dto.PartETag	uploadPart(String objectKey, File file, String uploadId, int partId, long partSize)

从类继承的方法 java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

字段详细资料

ks3Client

public com.ksyun.ks3.service.Ks3 ks3Client

bucket

public String bucket

context

public FileSystemStoreContext context

parallel_delete_pool_size

public int parallel_delete_pool_size

parallel_delete_thread_size

public int parallel_delete_thread_size

parallel_delete_speed_limit

public int parallel_delete_speed_limit

parallel_copy_pool_size

public int parallel_copy_pool_size

parallel_copy_thread_size

public int parallel_copy_thread_size

parallel_copy_speed_limit

public int parallel_copy_speed_limit

parallel_copy_part_pool_size

public int parallel_copy_part_pool_size

parallel_copy_part_thread_size

public int parallel_copy_part_thread_size

parallel_copy_part_speed_limit

public int parallel_copy_part_speed_limit

parallel_commit_pool_size

public int parallel_commit_pool_size

parallel_commit_thread_size

public int parallel_commit_thread_size

parallel_commit_speed_limit

public int parallel_commit_speed_limit

parallel_unser_pool_size

public int parallel_unser_pool_size

parallel_unser_thread_size

public int parallel_unser_thread_size

parallel_unser_speed_limit

public int parallel_unser_speed_limit

greater_than_this_use_parallel_when_destroy

public int greater_than_this_use_parallel_when_destroy

parallel_upload_part_pool_size

public int parallel_upload_part_pool_size

parallel_upload_part_thread_size

public int parallel_upload_part_thread_size

parallel_upload_part_limit

public int parallel_upload_part_limit

blockSize

public int blockSize

构造器详细资料

Ks3FileSystemStore

public Ks3FileSystemStore()

方法详细资料

getBlockSize

public int getBlockSize()

getStatistics

public org.apache.hadoop.fs.FileSystem.Statistics getStatistics()

getBucketAndEndpoint

public static org.apache.commons.lang3.tuple.Pair<String,String> getBucketAndEndpoint(URI uri,
                                                                                      String defaultBucket)

initialize

public void initialize(URI uri,
                       org.apache.hadoop.fs.Path workingDir,
                       org.apache.hadoop.fs.FileSystem.Statistics statistics,
                       org.apache.hadoop.conf.Configuration conf)

initParallelConfig

public void initParallelConfig()

initContext

public void initContext()

getCopyBlockSize

public int getCopyBlockSize()

getOverallCopyMaxLen

public long getOverallCopyMaxLen()

renewKs3Client

public void renewKs3Client()

getMetadata

public com.ksyun.ks3.dto.ObjectMetadata getMetadata(String objectKey)

getMetadata

public com.ksyun.ks3.dto.ObjectMetadata getMetadata(String objectKey,
                                                    com.google.common.util.concurrent.RateLimiter rateLimiter)

listAllSubPaths

public ListObjectsResult listAllSubPaths(String objectKey)

listAllSubPaths

public ListObjectsResult listAllSubPaths(String objectKey,
                                         int limit)

listAllSubPaths

public ListObjectsResult listAllSubPaths(String objectKey,
                                         int limit,
                                         int pageNum)

listAllObjects

public ListObjectsResult listAllObjects(String objectKey)

listAllObjects

public ListObjectsResult listAllObjects(String objectKey,
                                        int limit)

listAllObjects

public ListObjectsResult listAllObjects(String objectKey,
                                        int limit,
                                        int pageNum)

getObject

public com.ksyun.ks3.AutoAbortInputStream getObject(String objectKey,
                                                    long contentLength,
                                                    long pos)
                                             throws EOFException

抛出:

EOFException

getObject

public com.ksyun.ks3.AutoAbortInputStream getObject(String objectKey)

getObject

public com.ksyun.ks3.AutoAbortInputStream getObject(String objectKey,
                                                    com.google.common.util.concurrent.RateLimiter rateLimiter)

putObject

public void putObject(String objectKey,
                      File file)

putObject

public void putObject(String objectKey,
                      byte[] bytes)

putObject

public void putObject(Ks3BlockBuffer ks3BlockBuffer,
                      int blockLength)

deleteDir

public void deleteDir(String key,
                      boolean skip404)

deleteObject

public void deleteObject(String objectKey)

deleteObject

public void deleteObject(String objectKey,
                         com.google.common.util.concurrent.RateLimiter rateLimiter)

deleteObjectSkip404

public void deleteObjectSkip404(String objectKey)

deleteObjectSkip404

public void deleteObjectSkip404(String objectKey,
                                com.google.common.util.concurrent.RateLimiter rateLimiter)

deleteObjects

public void deleteObjects(List<String> objectKeys)

serialDeleteObjects

public void serialDeleteObjects(List<String> objectKeys)

parallelDeleteObjects

public void parallelDeleteObjects(List<String> objectKeys)

parallelDeleteObjects

public void parallelDeleteObjects(Ks3FileSystemStore.EngineSender runnable)

parallelDeleteObjects

public void parallelDeleteObjects(Ks3FileSystemStore.EngineSender runnable,
                                  int rateLimit)

copyObject

public void copyObject(String srcObjectKey,
                       String dstObjectKey)

copyObject

public void copyObject(com.ksyun.ks3.dto.ObjectMetadata srcMeta,
                       String srcObjectKey,
                       String dstObjectKey)

copyObject

public void copyObject(com.ksyun.ks3.dto.ObjectMetadata srcMeta,
                       String srcObjectKey,
                       String dstObjectKey,
                       com.google.common.util.concurrent.RateLimiter rateLimiter)

copyPart

public void copyPart(com.google.common.util.concurrent.RateLimiter rateLimiter,
                     CopyPartBean bean)

copyObjects

public void copyObjects(List<org.apache.commons.lang3.tuple.Pair<String,String>> keys)

createEmptyObject

public void createEmptyObject(String objectKey)

createEmptyObject

public void createEmptyObject(String objectKey,
                              com.google.common.util.concurrent.RateLimiter rateLimiter)

initMultipartUpload

public String initMultipartUpload(String objectKey)

initMultipartUpload

public String initMultipartUpload(String objectKey,
                                  com.google.common.util.concurrent.RateLimiter rateLimiter)

uploadPart

public com.ksyun.ks3.dto.PartETag uploadPart(Ks3BlockBuffer ks3BlockBuffer,
                                             String uploadId,
                                             com.google.common.util.concurrent.RateLimiter rateLimiter,
                                             int blockLength)
                                      throws com.ksyun.ks3.exception.Ks3ClientException,
                                             com.ksyun.ks3.exception.Ks3ServiceException

抛出:

com.ksyun.ks3.exception.Ks3ClientException

com.ksyun.ks3.exception.Ks3ServiceException

uploadPart

public com.ksyun.ks3.dto.PartETag uploadPart(String objectKey,
                                             File file,
                                             String uploadId,
                                             int partId,
                                             long partSize)

completeMultipartUpload

public void completeMultipartUpload(String objectKey,
                                    String uploadId,
                                    List<com.ksyun.ks3.dto.PartETag> eTags)

completeMultipartUpload

public void completeMultipartUpload(String objectKey,
                                    String uploadId,
                                    List<com.ksyun.ks3.dto.PartETag> eTags,
                                    com.google.common.util.concurrent.RateLimiter rateLimiter)

getConf

public org.apache.hadoop.conf.Configuration getConf()

checkIsRequestCompress

public void checkIsRequestCompress(com.ksyun.ks3.service.request.Ks3WebServiceRequest request)

pathToKey

public String pathToKey(org.apache.hadoop.fs.Path path)