001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.hdfs;
018
019 import java.io.IOException;
020 import java.util.List;
021 import java.util.concurrent.ScheduledExecutorService;
022 import java.util.concurrent.TimeUnit;
023 import java.util.concurrent.atomic.AtomicBoolean;
024
025 import org.apache.camel.Exchange;
026 import org.apache.camel.impl.DefaultProducer;
027
028 public class HdfsProducer extends DefaultProducer {
029
030 private final HdfsConfiguration config;
031 private final StringBuilder hdfsPath;
032 private final AtomicBoolean idle = new AtomicBoolean(false);
033 private ScheduledExecutorService scheduler;
034 private HdfsOutputStream ostream;
035 private long splitNum;
036
037 public static final class SplitStrategy {
038 private SplitStrategyType type;
039 private long value;
040
041 public SplitStrategy(SplitStrategyType type, long value) {
042 this.type = type;
043 this.value = value;
044 }
045
046 public SplitStrategyType getType() {
047 return type;
048 }
049
050 public long getValue() {
051 return value;
052 }
053 }
054
055 public enum SplitStrategyType {
056 BYTES {
057 @Override
058 public boolean split(HdfsOutputStream oldOstream, long value, HdfsProducer producer) {
059 return oldOstream.getNumOfWrittenBytes() >= value;
060 }
061 },
062
063 MESSAGES {
064 @Override
065 public boolean split(HdfsOutputStream oldOstream, long value, HdfsProducer producer) {
066 return oldOstream.getNumOfWrittenMessages() >= value;
067 }
068 },
069
070 IDLE {
071 @Override
072 public boolean split(HdfsOutputStream oldOstream, long value, HdfsProducer producer) {
073 return producer.idle.get();
074 }
075 };
076
077 public abstract boolean split(HdfsOutputStream oldOstream, long value, HdfsProducer producer);
078 }
079
080 public HdfsProducer(HdfsEndpoint endpoint, HdfsConfiguration config) {
081 super(endpoint);
082 this.config = config;
083 this.hdfsPath = config.getFileSystemType().getHdfsPath(config);
084 }
085
086 @Override
087 protected void doStart() throws Exception {
088 super.doStart();
089 StringBuilder actualPath = new StringBuilder(hdfsPath);
090 if (config.getSplitStrategies().size() > 0) {
091 actualPath = newFileName();
092 }
093 ostream = HdfsOutputStream.createOutputStream(actualPath.toString(), config);
094
095 SplitStrategy idleStrategy = null;
096 for (SplitStrategy strategy : config.getSplitStrategies()) {
097 if (strategy.type == SplitStrategyType.IDLE) {
098 idleStrategy = strategy;
099 break;
100 }
101 }
102 if (idleStrategy != null) {
103 scheduler = getEndpoint().getCamelContext().getExecutorServiceStrategy().newScheduledThreadPool(this, "IdleCheck", 1);
104 log.debug("Creating IdleCheck task scheduled to run every {} millis", config.getCheckIdleInterval());
105 scheduler.scheduleAtFixedRate(new IdleCheck(idleStrategy), 1000, config.getCheckIdleInterval(), TimeUnit.MILLISECONDS);
106 }
107 }
108
109 @Override
110 protected void doStop() throws Exception {
111 super.doStop();
112 ostream.close();
113 if (scheduler != null) {
114 getEndpoint().getCamelContext().getExecutorServiceStrategy().shutdown(scheduler);
115 scheduler = null;
116 }
117 }
118
119 @Override
120 public void process(Exchange exchange) throws Exception {
121 Object body = exchange.getIn().getBody();
122 Object key = exchange.getIn().getHeader(HdfsHeader.KEY.name());
123
124 boolean split = false;
125 List<SplitStrategy> strategies = config.getSplitStrategies();
126 for (SplitStrategy splitStrategy : strategies) {
127 split |= splitStrategy.getType().split(ostream, splitStrategy.value, this);
128 }
129
130 if (split) {
131 ostream.close();
132 StringBuilder actualPath = newFileName();
133 ostream = HdfsOutputStream.createOutputStream(actualPath.toString(), config);
134 }
135 ostream.append(key, body, exchange.getContext().getTypeConverter());
136 idle.set(false);
137 }
138
139 public HdfsOutputStream getOstream() {
140 return ostream;
141 }
142
143 private StringBuilder newFileName() {
144 StringBuilder actualPath = new StringBuilder(hdfsPath);
145 actualPath.append(splitNum);
146 splitNum++;
147 return actualPath;
148 }
149
150 public final AtomicBoolean isIdle() {
151 return idle;
152 }
153
154 /**
155 * Idle check background task
156 */
157 private final class IdleCheck implements Runnable {
158
159 private final SplitStrategy strategy;
160
161 private IdleCheck(SplitStrategy strategy) {
162 this.strategy = strategy;
163 }
164
165 @Override
166 public void run() {
167 HdfsProducer.this.log.trace("IdleCheck running");
168
169 if (System.currentTimeMillis() - ostream.getLastAccess() > strategy.value && !idle.get() && !ostream.isBusy().get()) {
170 idle.set(true);
171 try {
172 ostream.close();
173 } catch (IOException e) {
174 // ignore
175 }
176 }
177 }
178
179 @Override
180 public String toString() {
181 return "IdleCheck";
182 }
183 }
184 }
185