001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.hdfs;
018
019 import java.net.URI;
020 import java.net.URISyntaxException;
021 import java.util.ArrayList;
022 import java.util.List;
023 import java.util.Map;
024
025 import org.apache.camel.util.URISupport;
026 import org.apache.hadoop.io.SequenceFile;
027
028 public class HdfsConfiguration {
029
030 private URI uri;
031 private String hostName;
032 private int port = HdfsConstants.DEFAULT_PORT;
033 private String path;
034 private boolean overwrite = true;
035 private boolean append;
036 private int bufferSize = HdfsConstants.DEFAULT_BUFFERSIZE;
037 private short replication = HdfsConstants.DEFAULT_REPLICATION;
038 private long blockSize = HdfsConstants.DEFAULT_BLOCKSIZE;
039 private SequenceFile.CompressionType compressionType = HdfsConstants.DEFAULT_COMPRESSIONTYPE;
040 private HdfsCompressionCodec compressionCodec = HdfsConstants.DEFAULT_CODEC;
041 private HdfsFileType fileType = HdfsFileType.NORMAL_FILE;
042 private HdfsFileSystemType fileSystemType = HdfsFileSystemType.HDFS;
043 private HdfsWritableFactories.WritableType keyType = HdfsWritableFactories.WritableType.NULL;
044 private HdfsWritableFactories.WritableType valueType = HdfsWritableFactories.WritableType.BYTES;
045 private String openedSuffix = HdfsConstants.DEFAULT_OPENED_SUFFIX;
046 private String readSuffix = HdfsConstants.DEFAULT_READ_SUFFIX;
047 private long initialDelay;
048 private long delay = HdfsConstants.DEFAULT_DELAY;
049 private String pattern = HdfsConstants.DEFAULT_PATTERN;
050 private int chunkSize = HdfsConstants.DEFAULT_BUFFERSIZE;
051 private int checkIdleInterval = HdfsConstants.DEFAULT_CHECK_IDLE_INTERVAL;
052 private List<HdfsProducer.SplitStrategy> splitStrategies;
053
054 public HdfsConfiguration() {
055 }
056
057 private Boolean getBoolean(Map hdfsSettings, String param, Boolean dflt) {
058 if (hdfsSettings.containsKey(param)) {
059 return Boolean.valueOf((String) hdfsSettings.get(param));
060 } else {
061 return dflt;
062 }
063 }
064
065 private Integer getInteger(Map hdfsSettings, String param, Integer dflt) {
066 if (hdfsSettings.containsKey(param)) {
067 return Integer.valueOf((String) hdfsSettings.get(param));
068 } else {
069 return dflt;
070 }
071 }
072
073 private Short getShort(Map hdfsSettings, String param, Short dflt) {
074 if (hdfsSettings.containsKey(param)) {
075 return Short.valueOf((String) hdfsSettings.get(param));
076 } else {
077 return dflt;
078 }
079 }
080
081 private Long getLong(Map hdfsSettings, String param, Long dflt) {
082 if (hdfsSettings.containsKey(param)) {
083 return Long.valueOf((String) hdfsSettings.get(param));
084 } else {
085 return dflt;
086 }
087 }
088
089 private HdfsFileType getFileType(Map hdfsSettings, String param, HdfsFileType dflt) {
090 String eit = (String) hdfsSettings.get(param);
091 if (eit != null) {
092 return HdfsFileType.valueOf(eit);
093 } else {
094 return dflt;
095 }
096 }
097
098 private HdfsFileSystemType getFileSystemType(Map hdfsSettings, String param, HdfsFileSystemType dflt) {
099 String eit = (String) hdfsSettings.get(param);
100 if (eit != null) {
101 return HdfsFileSystemType.valueOf(eit);
102 } else {
103 return dflt;
104 }
105 }
106
107 private HdfsWritableFactories.WritableType getWritableType(Map hdfsSettings, String param, HdfsWritableFactories.WritableType dflt) {
108 String eit = (String) hdfsSettings.get(param);
109 if (eit != null) {
110 return HdfsWritableFactories.WritableType.valueOf(eit);
111 } else {
112 return dflt;
113 }
114 }
115
116 private SequenceFile.CompressionType getCompressionType(Map hdfsSettings, String param, SequenceFile.CompressionType ct) {
117 String eit = (String) hdfsSettings.get(param);
118 if (eit != null) {
119 return SequenceFile.CompressionType.valueOf(eit);
120 } else {
121 return ct;
122 }
123 }
124
125 private HdfsCompressionCodec getCompressionCodec(Map hdfsSettings, String param, HdfsCompressionCodec cd) {
126 String eit = (String) hdfsSettings.get(param);
127 if (eit != null) {
128 return HdfsCompressionCodec.valueOf(eit);
129 } else {
130 return cd;
131 }
132 }
133
134 private String getString(Map hdfsSettings, String param, String dflt) {
135 if (hdfsSettings.containsKey(param)) {
136 return (String) hdfsSettings.get(param);
137 } else {
138 return dflt;
139 }
140 }
141
142 private List<HdfsProducer.SplitStrategy> getSplitStrategies(Map hdfsSettings) {
143 List<HdfsProducer.SplitStrategy> strategies = new ArrayList<HdfsProducer.SplitStrategy>();
144 for (Object obj : hdfsSettings.keySet()) {
145 String key = (String) obj;
146 if ("splitStrategy".equals(key)) {
147 String eit = (String) hdfsSettings.get(key);
148 if (eit != null) {
149 String[] strstrategies = eit.split(",");
150 for (int i = 0; i < strstrategies.length; i++) {
151 String strstrategy = strstrategies[i];
152 String tokens[] = strstrategy.split(":");
153 if (tokens.length != 2) {
154 throw new IllegalArgumentException("Wrong Split Strategy " + key + "=" + eit);
155 }
156 HdfsProducer.SplitStrategyType sst = HdfsProducer.SplitStrategyType.valueOf(tokens[0]);
157 long ssv = Long.valueOf(tokens[1]);
158 strategies.add(new HdfsProducer.SplitStrategy(sst, ssv));
159 }
160 }
161 }
162 }
163 return strategies;
164 }
165
166 public void checkConsumerOptions() {
167 }
168
169 public void checkProducerOptions() {
170 if (isAppend()) {
171 if (getSplitStrategies().size() != 0) {
172 throw new IllegalArgumentException("Split Strategies incompatible with append=true");
173 }
174 if (getFileType() != HdfsFileType.NORMAL_FILE) {
175 throw new IllegalArgumentException("append=true works only with NORMAL_FILEs");
176 }
177 }
178 }
179
180 public void parseURI(URI uri) throws URISyntaxException {
181 String protocol = uri.getScheme();
182 if (!protocol.equalsIgnoreCase("hdfs")) {
183 throw new IllegalArgumentException("Unrecognized Cache protocol: " + protocol + " for uri: " + uri);
184 }
185 hostName = uri.getHost();
186 port = uri.getPort() == -1 ? HdfsConstants.DEFAULT_PORT : uri.getPort();
187 path = uri.getPath();
188 Map hdfsSettings = URISupport.parseParameters(uri);
189
190 overwrite = getBoolean(hdfsSettings, "overwrite", overwrite);
191 append = getBoolean(hdfsSettings, "append", append);
192 bufferSize = getInteger(hdfsSettings, "bufferSize", bufferSize);
193 replication = getShort(hdfsSettings, "replication", replication);
194 blockSize = getLong(hdfsSettings, "blockSize", blockSize);
195 compressionType = getCompressionType(hdfsSettings, "compressionType", compressionType);
196 compressionCodec = getCompressionCodec(hdfsSettings, "compressionCodec", compressionCodec);
197 fileType = getFileType(hdfsSettings, "fileType", fileType);
198 fileSystemType = getFileSystemType(hdfsSettings, "fileSystemType", fileSystemType);
199 keyType = getWritableType(hdfsSettings, "keyType", keyType);
200 valueType = getWritableType(hdfsSettings, "valueType", valueType);
201 openedSuffix = getString(hdfsSettings, "openedSuffix", openedSuffix);
202 readSuffix = getString(hdfsSettings, "readSuffix", readSuffix);
203 initialDelay = getLong(hdfsSettings, "initialDelay", initialDelay);
204 delay = getLong(hdfsSettings, "delay", delay);
205 pattern = getString(hdfsSettings, "pattern", pattern);
206 chunkSize = getInteger(hdfsSettings, "chunkSize", chunkSize);
207 splitStrategies = getSplitStrategies(hdfsSettings);
208 }
209
210 public URI getUri() {
211 return uri;
212 }
213
214 public void setUri(URI uri) {
215 this.uri = uri;
216 }
217
218 public String getHostName() {
219 return hostName;
220 }
221
222 public void setHostName(String hostName) {
223 this.hostName = hostName;
224 }
225
226 public int getPort() {
227 return port;
228 }
229
230 public void setPort(int port) {
231 this.port = port;
232 }
233
234 public String getPath() {
235 return path;
236 }
237
238 public void setPath(String path) {
239 this.path = path;
240 }
241
242 public boolean isOverwrite() {
243 return overwrite;
244 }
245
246 public void setOverwrite(boolean overwrite) {
247 this.overwrite = overwrite;
248 }
249
250 public boolean isAppend() {
251 return append;
252 }
253
254 public void setAppend(boolean append) {
255 this.append = append;
256 }
257
258 public int getBufferSize() {
259 return bufferSize;
260 }
261
262 public void setBufferSize(int bufferSize) {
263 this.bufferSize = bufferSize;
264 }
265
266 public short getReplication() {
267 return replication;
268 }
269
270 public void setReplication(short replication) {
271 this.replication = replication;
272 }
273
274 public long getBlockSize() {
275 return blockSize;
276 }
277
278 public void setBlockSize(long blockSize) {
279 this.blockSize = blockSize;
280 }
281
282 public HdfsFileType getFileType() {
283 return fileType;
284 }
285
286 public void setFileType(HdfsFileType fileType) {
287 this.fileType = fileType;
288 }
289
290 public SequenceFile.CompressionType getCompressionType() {
291 return compressionType;
292 }
293
294 public void setCompressionType(SequenceFile.CompressionType compressionType) {
295 this.compressionType = compressionType;
296 }
297
298 public HdfsCompressionCodec getCompressionCodec() {
299 return compressionCodec;
300 }
301
302 public void setCompressionCodec(HdfsCompressionCodec compressionCodec) {
303 this.compressionCodec = compressionCodec;
304 }
305
306 public void setFileSystemType(HdfsFileSystemType fileSystemType) {
307 this.fileSystemType = fileSystemType;
308 }
309
310 public HdfsFileSystemType getFileSystemType() {
311 return fileSystemType;
312 }
313
314 public HdfsWritableFactories.WritableType getKeyType() {
315 return keyType;
316 }
317
318 public void setKeyType(HdfsWritableFactories.WritableType keyType) {
319 this.keyType = keyType;
320 }
321
322 public HdfsWritableFactories.WritableType getValueType() {
323 return valueType;
324 }
325
326 public void setValueType(HdfsWritableFactories.WritableType valueType) {
327 this.valueType = valueType;
328 }
329
330 public void setOpenedSuffix(String openedSuffix) {
331 this.openedSuffix = openedSuffix;
332 }
333
334 public String getOpenedSuffix() {
335 return openedSuffix;
336 }
337
338 public void setReadSuffix(String readSuffix) {
339 this.readSuffix = readSuffix;
340 }
341
342 public String getReadSuffix() {
343 return readSuffix;
344 }
345
346 public void setInitialDelay(long initialDelay) {
347 this.initialDelay = initialDelay;
348 }
349
350 public long getInitialDelay() {
351 return initialDelay;
352 }
353
354 public void setDelay(long delay) {
355 this.delay = delay;
356 }
357
358 public long getDelay() {
359 return delay;
360 }
361
362 public void setPattern(String pattern) {
363 this.pattern = pattern;
364 }
365
366 public String getPattern() {
367 return pattern;
368 }
369
370 public void setChunkSize(int chunkSize) {
371 this.chunkSize = chunkSize;
372 }
373
374 public int getChunkSize() {
375 return chunkSize;
376 }
377
378 public void setCheckIdleInterval(int checkIdleInterval) {
379 this.checkIdleInterval = checkIdleInterval;
380 }
381
382 public int getCheckIdleInterval() {
383 return checkIdleInterval;
384 }
385
386 public List<HdfsProducer.SplitStrategy> getSplitStrategies() {
387 return splitStrategies;
388 }
389
390 public void setSplitStrategy(String splitStrategy) {
391 // noop
392 }
393 }