001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.datanode.web.resources;
019
020 import java.io.IOException;
021 import java.io.InputStream;
022 import java.net.URI;
023 import java.net.URISyntaxException;
024 import java.security.PrivilegedExceptionAction;
025 import java.util.EnumSet;
026
027 import javax.servlet.ServletContext;
028 import javax.servlet.http.HttpServletRequest;
029 import javax.servlet.http.HttpServletResponse;
030 import javax.ws.rs.Consumes;
031 import javax.ws.rs.DefaultValue;
032 import javax.ws.rs.GET;
033 import javax.ws.rs.POST;
034 import javax.ws.rs.PUT;
035 import javax.ws.rs.Path;
036 import javax.ws.rs.PathParam;
037 import javax.ws.rs.Produces;
038 import javax.ws.rs.QueryParam;
039 import javax.ws.rs.core.Context;
040 import javax.ws.rs.core.MediaType;
041 import javax.ws.rs.core.Response;
042
043 import com.google.common.annotations.VisibleForTesting;
044 import org.apache.commons.logging.Log;
045 import org.apache.commons.logging.LogFactory;
046 import org.apache.hadoop.conf.Configuration;
047 import org.apache.hadoop.fs.CreateFlag;
048 import org.apache.hadoop.fs.FSDataOutputStream;
049 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
050 import org.apache.hadoop.fs.permission.FsPermission;
051 import org.apache.hadoop.hdfs.DFSClient;
052 import org.apache.hadoop.hdfs.HAUtil;
053 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
054 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
055 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
056 import org.apache.hadoop.hdfs.server.datanode.DataNode;
057 import org.apache.hadoop.hdfs.web.JsonUtil;
058 import org.apache.hadoop.hdfs.web.ParamFilter;
059 import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
060 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
061 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
062 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
063 import org.apache.hadoop.hdfs.web.resources.DelegationParam;
064 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
065 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
066 import org.apache.hadoop.hdfs.web.resources.LengthParam;
067 import org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
068 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
069 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
070 import org.apache.hadoop.hdfs.web.resources.Param;
071 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
072 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
073 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
074 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
075 import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
076 import org.apache.hadoop.io.IOUtils;
077 import org.apache.hadoop.security.SecurityUtil;
078 import org.apache.hadoop.security.UserGroupInformation;
079 import org.apache.hadoop.security.token.Token;
080
081 import com.sun.jersey.spi.container.ResourceFilters;
082
083 /** Web-hdfs DataNode implementation. */
084 @Path("")
085 @ResourceFilters(ParamFilter.class)
086 public class DatanodeWebHdfsMethods {
087 public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
088
089 private static final UriFsPathParam ROOT = new UriFsPathParam("");
090
091 private @Context ServletContext context;
092 private @Context HttpServletRequest request;
093 private @Context HttpServletResponse response;
094
095 private void init(final UserGroupInformation ugi,
096 final DelegationParam delegation, final String nnId,
097 final UriFsPathParam path, final HttpOpParam<?> op,
098 final Param<?, ?>... parameters) throws IOException {
099 if (LOG.isTraceEnabled()) {
100 LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
101 + ", ugi=" + ugi + Param.toSortedString(", ", parameters));
102 }
103 if (nnId == null) {
104 throw new IllegalArgumentException(NamenodeAddressParam.NAME
105 + " is not specified.");
106 }
107
108 //clear content type
109 response.setContentType(null);
110
111 if (UserGroupInformation.isSecurityEnabled()) {
112 //add a token for RPC.
113 final Token<DelegationTokenIdentifier> token = deserializeToken
114 (delegation.getValue(), nnId);
115 ugi.addToken(token);
116 }
117 }
118
119 @VisibleForTesting
120 Token<DelegationTokenIdentifier> deserializeToken
121 (String delegation,String nnId) throws IOException {
122 final DataNode datanode = (DataNode) context.getAttribute("datanode");
123 final Configuration conf = datanode.getConf();
124 final Token<DelegationTokenIdentifier> token = new
125 Token<DelegationTokenIdentifier>();
126 token.decodeFromUrlString(delegation);
127 URI nnUri = URI.create(HdfsConstants.HDFS_URI_SCHEME +
128 "://" + nnId);
129 boolean isLogical = HAUtil.isLogicalUri(conf, nnUri);
130 if (isLogical) {
131 token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri,
132 HdfsConstants.HDFS_URI_SCHEME));
133 } else {
134 token.setService(SecurityUtil.buildTokenService(nnUri));
135 }
136 token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
137 return token;
138 }
139
140 /** Handle HTTP PUT request for the root. */
141 @PUT
142 @Path("/")
143 @Consumes({"*/*"})
144 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
145 public Response putRoot(
146 final InputStream in,
147 @Context final UserGroupInformation ugi,
148 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
149 final DelegationParam delegation,
150 @QueryParam(NamenodeAddressParam.NAME)
151 @DefaultValue(NamenodeAddressParam.DEFAULT)
152 final NamenodeAddressParam namenode,
153 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
154 final PutOpParam op,
155 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
156 final PermissionParam permission,
157 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
158 final OverwriteParam overwrite,
159 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
160 final BufferSizeParam bufferSize,
161 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
162 final ReplicationParam replication,
163 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
164 final BlockSizeParam blockSize
165 ) throws IOException, InterruptedException {
166 return put(in, ugi, delegation, namenode, ROOT, op, permission,
167 overwrite, bufferSize, replication, blockSize);
168 }
169
170 /** Handle HTTP PUT request. */
171 @PUT
172 @Path("{" + UriFsPathParam.NAME + ":.*}")
173 @Consumes({"*/*"})
174 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
175 public Response put(
176 final InputStream in,
177 @Context final UserGroupInformation ugi,
178 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
179 final DelegationParam delegation,
180 @QueryParam(NamenodeAddressParam.NAME)
181 @DefaultValue(NamenodeAddressParam.DEFAULT)
182 final NamenodeAddressParam namenode,
183 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
184 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
185 final PutOpParam op,
186 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
187 final PermissionParam permission,
188 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
189 final OverwriteParam overwrite,
190 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
191 final BufferSizeParam bufferSize,
192 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
193 final ReplicationParam replication,
194 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
195 final BlockSizeParam blockSize
196 ) throws IOException, InterruptedException {
197
198 final String nnId = namenode.getValue();
199 init(ugi, delegation, nnId, path, op, permission,
200 overwrite, bufferSize, replication, blockSize);
201
202 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
203 @Override
204 public Response run() throws IOException, URISyntaxException {
205 return put(in, nnId, path.getAbsolutePath(), op,
206 permission, overwrite, bufferSize, replication, blockSize);
207 }
208 });
209 }
210
211 private Response put(
212 final InputStream in,
213 final String nnId,
214 final String fullpath,
215 final PutOpParam op,
216 final PermissionParam permission,
217 final OverwriteParam overwrite,
218 final BufferSizeParam bufferSize,
219 final ReplicationParam replication,
220 final BlockSizeParam blockSize
221 ) throws IOException, URISyntaxException {
222 final DataNode datanode = (DataNode)context.getAttribute("datanode");
223
224 switch(op.getValue()) {
225 case CREATE:
226 {
227 final Configuration conf = new Configuration(datanode.getConf());
228 conf.set(FsPermission.UMASK_LABEL, "000");
229
230 final int b = bufferSize.getValue(conf);
231 DFSClient dfsclient = newDfsClient(nnId, conf);
232 FSDataOutputStream out = null;
233 try {
234 out = dfsclient.createWrappedOutputStream(dfsclient.create(
235 fullpath, permission.getFsPermission(),
236 overwrite.getValue() ?
237 EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) :
238 EnumSet.of(CreateFlag.CREATE),
239 replication.getValue(conf), blockSize.getValue(conf), null,
240 b, null), null);
241 IOUtils.copyBytes(in, out, b);
242 out.close();
243 out = null;
244 dfsclient.close();
245 dfsclient = null;
246 } finally {
247 IOUtils.cleanup(LOG, out);
248 IOUtils.cleanup(LOG, dfsclient);
249 }
250 final String scheme = "http".equals(request.getScheme()) ?
251 WebHdfsFileSystem.SCHEME : SWebHdfsFileSystem.SCHEME;
252 final URI uri = new URI(scheme, nnId, fullpath, null, null);
253 return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
254 }
255 default:
256 throw new UnsupportedOperationException(op + " is not supported");
257 }
258 }
259
260 /** Handle HTTP POST request for the root for the root. */
261 @POST
262 @Path("/")
263 @Consumes({"*/*"})
264 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
265 public Response postRoot(
266 final InputStream in,
267 @Context final UserGroupInformation ugi,
268 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
269 final DelegationParam delegation,
270 @QueryParam(NamenodeAddressParam.NAME)
271 @DefaultValue(NamenodeAddressParam.DEFAULT)
272 final NamenodeAddressParam namenode,
273 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
274 final PostOpParam op,
275 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
276 final BufferSizeParam bufferSize
277 ) throws IOException, InterruptedException {
278 return post(in, ugi, delegation, namenode, ROOT, op, bufferSize);
279 }
280
281 /** Handle HTTP POST request. */
282 @POST
283 @Path("{" + UriFsPathParam.NAME + ":.*}")
284 @Consumes({"*/*"})
285 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
286 public Response post(
287 final InputStream in,
288 @Context final UserGroupInformation ugi,
289 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
290 final DelegationParam delegation,
291 @QueryParam(NamenodeAddressParam.NAME)
292 @DefaultValue(NamenodeAddressParam.DEFAULT)
293 final NamenodeAddressParam namenode,
294 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
295 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
296 final PostOpParam op,
297 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
298 final BufferSizeParam bufferSize
299 ) throws IOException, InterruptedException {
300
301 final String nnId = namenode.getValue();
302 init(ugi, delegation, nnId, path, op, bufferSize);
303
304 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
305 @Override
306 public Response run() throws IOException {
307 return post(in, nnId, path.getAbsolutePath(), op,
308 bufferSize);
309 }
310 });
311 }
312
313 private Response post(
314 final InputStream in,
315 final String nnId,
316 final String fullpath,
317 final PostOpParam op,
318 final BufferSizeParam bufferSize
319 ) throws IOException {
320 final DataNode datanode = (DataNode)context.getAttribute("datanode");
321
322 switch(op.getValue()) {
323 case APPEND:
324 {
325 final Configuration conf = new Configuration(datanode.getConf());
326 final int b = bufferSize.getValue(conf);
327 DFSClient dfsclient = newDfsClient(nnId, conf);
328 FSDataOutputStream out = null;
329 try {
330 out = dfsclient.append(fullpath, b, null, null);
331 IOUtils.copyBytes(in, out, b);
332 out.close();
333 out = null;
334 dfsclient.close();
335 dfsclient = null;
336 } finally {
337 IOUtils.cleanup(LOG, out);
338 IOUtils.cleanup(LOG, dfsclient);
339 }
340 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
341 }
342 default:
343 throw new UnsupportedOperationException(op + " is not supported");
344 }
345 }
346
347 /** Handle HTTP GET request for the root. */
348 @GET
349 @Path("/")
350 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
351 public Response getRoot(
352 @Context final UserGroupInformation ugi,
353 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
354 final DelegationParam delegation,
355 @QueryParam(NamenodeAddressParam.NAME)
356 @DefaultValue(NamenodeAddressParam.DEFAULT)
357 final NamenodeAddressParam namenode,
358 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
359 final GetOpParam op,
360 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
361 final OffsetParam offset,
362 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
363 final LengthParam length,
364 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
365 final BufferSizeParam bufferSize
366 ) throws IOException, InterruptedException {
367 return get(ugi, delegation, namenode, ROOT, op, offset, length,
368 bufferSize);
369 }
370
371 /** Handle HTTP GET request. */
372 @GET
373 @Path("{" + UriFsPathParam.NAME + ":.*}")
374 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
375 public Response get(
376 @Context final UserGroupInformation ugi,
377 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
378 final DelegationParam delegation,
379 @QueryParam(NamenodeAddressParam.NAME)
380 @DefaultValue(NamenodeAddressParam.DEFAULT)
381 final NamenodeAddressParam namenode,
382 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
383 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
384 final GetOpParam op,
385 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
386 final OffsetParam offset,
387 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
388 final LengthParam length,
389 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
390 final BufferSizeParam bufferSize
391 ) throws IOException, InterruptedException {
392
393 final String nnId = namenode.getValue();
394 init(ugi, delegation, nnId, path, op, offset, length, bufferSize);
395
396 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
397 @Override
398 public Response run() throws IOException {
399 return get(nnId, path.getAbsolutePath(), op, offset,
400 length, bufferSize);
401 }
402 });
403 }
404
405 private Response get(
406 final String nnId,
407 final String fullpath,
408 final GetOpParam op,
409 final OffsetParam offset,
410 final LengthParam length,
411 final BufferSizeParam bufferSize
412 ) throws IOException {
413 final DataNode datanode = (DataNode)context.getAttribute("datanode");
414 final Configuration conf = new Configuration(datanode.getConf());
415
416 switch(op.getValue()) {
417 case OPEN:
418 {
419 final int b = bufferSize.getValue(conf);
420 final DFSClient dfsclient = newDfsClient(nnId, conf);
421 HdfsDataInputStream in = null;
422 try {
423 in = dfsclient.createWrappedInputStream(
424 dfsclient.open(fullpath, b, true));
425 in.seek(offset.getValue());
426 } catch(IOException ioe) {
427 IOUtils.cleanup(LOG, in);
428 IOUtils.cleanup(LOG, dfsclient);
429 throw ioe;
430 }
431
432 final long n = length.getValue() != null ?
433 Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
434 in.getVisibleLength() - offset.getValue();
435
436 // jetty 6 reserves 12 bytes in the out buffer for chunked responses
437 // (file length > 2GB) which causes extremely poor performance when
438 // 12 bytes of the output spill into another buffer which results
439 // in a big and little write
440 int outBufferSize = response.getBufferSize();
441 if (n > Integer.MAX_VALUE) {
442 outBufferSize -= 12;
443 }
444 /**
445 * Allow the Web UI to perform an AJAX request to get the data.
446 */
447 return Response.ok(new OpenEntity(in, n, outBufferSize, dfsclient))
448 .type(MediaType.APPLICATION_OCTET_STREAM)
449 .header("Access-Control-Allow-Methods", "GET")
450 .header("Access-Control-Allow-Origin", "*")
451 .build();
452 }
453 case GETFILECHECKSUM:
454 {
455 MD5MD5CRC32FileChecksum checksum = null;
456 DFSClient dfsclient = newDfsClient(nnId, conf);
457 try {
458 checksum = dfsclient.getFileChecksum(fullpath, Long.MAX_VALUE);
459 dfsclient.close();
460 dfsclient = null;
461 } finally {
462 IOUtils.cleanup(LOG, dfsclient);
463 }
464 final String js = JsonUtil.toJsonString(checksum);
465 return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
466 }
467 default:
468 throw new UnsupportedOperationException(op + " is not supported");
469 }
470 }
471
472 private static DFSClient newDfsClient(String nnId,
473 Configuration conf) throws IOException {
474 URI uri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://" + nnId);
475 return new DFSClient(uri, conf);
476 }
477 }