001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.hdfs;
018
019 import java.io.Closeable;
020 import java.io.IOException;
021 import java.util.concurrent.atomic.AtomicBoolean;
022 import java.util.concurrent.atomic.AtomicLong;
023
024 import org.apache.camel.RuntimeCamelException;
025 import org.apache.camel.TypeConverter;
026 import org.apache.hadoop.fs.Path;
027 import org.apache.hadoop.io.IOUtils;
028
029 public class HdfsOutputStream {
030
031 private HdfsFileType fileType;
032 private String actualPath;
033 private String suffixedPath;
034 private Closeable out;
035 private boolean opened;
036 private final AtomicLong numOfWrittenBytes = new AtomicLong(0L);
037 private final AtomicLong numOfWrittenMessages = new AtomicLong(0L);
038 private final AtomicLong lastAccess = new AtomicLong(Long.MAX_VALUE);
039 private final AtomicBoolean busy = new AtomicBoolean(false);
040
041 protected HdfsOutputStream() {
042 }
043
044 public static HdfsOutputStream createOutputStream(String hdfsPath, HdfsConfiguration configuration) throws IOException {
045 HdfsOutputStream ret = new HdfsOutputStream();
046 ret.fileType = configuration.getFileType();
047 ret.actualPath = hdfsPath;
048 HdfsInfo info = new HdfsInfo(ret.actualPath);
049
050 ret.suffixedPath = ret.actualPath + '.' + configuration.getOpenedSuffix();
051 if (configuration.isAppend()) {
052 if (!info.getFileSystem().exists(new Path(ret.actualPath))) {
053 configuration.setAppend(false);
054 } else {
055 info = new HdfsInfo(ret.suffixedPath);
056 info.getFileSystem().rename(new Path(ret.actualPath), new Path(ret.suffixedPath));
057 }
058 } else {
059 if (info.getFileSystem().exists(new Path(ret.actualPath))) {
060 if (configuration.isOverwrite()) {
061 info.getFileSystem().delete(new Path(ret.actualPath), true);
062 } else {
063 throw new RuntimeCamelException("The file already exists");
064 }
065 }
066 }
067 ret.out = ret.fileType.createOutputStream(ret.suffixedPath, configuration);
068 ret.opened = true;
069 return ret;
070 }
071
072 public void close() throws IOException {
073 if (opened) {
074 IOUtils.closeStream(out);
075 HdfsInfo info = new HdfsInfo(actualPath);
076 info.getFileSystem().rename(new Path(suffixedPath), new Path(actualPath));
077 opened = false;
078 }
079 }
080
081 public void append(Object key, Object value, TypeConverter typeConverter) {
082 try {
083 busy.set(true);
084 long nb = fileType.append(this, key, value, typeConverter);
085 numOfWrittenBytes.addAndGet(nb);
086 numOfWrittenMessages.incrementAndGet();
087 lastAccess.set(System.currentTimeMillis());
088 } finally {
089 busy.set(false);
090 }
091 }
092
093 public long getNumOfWrittenBytes() {
094 return numOfWrittenBytes.longValue();
095 }
096
097 public long getNumOfWrittenMessages() {
098 return numOfWrittenMessages.longValue();
099 }
100
101 public long getLastAccess() {
102 return lastAccess.longValue();
103 }
104
105 public String getActualPath() {
106 return actualPath;
107 }
108
109 public AtomicBoolean isBusy() {
110 return busy;
111 }
112
113 public Closeable getOut() {
114 return out;
115 }
116 }