001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.datanode.web.resources;
019
020 import java.io.IOException;
021 import java.io.InputStream;
022 import java.net.URI;
023 import java.net.URISyntaxException;
024 import java.security.PrivilegedExceptionAction;
025 import java.util.EnumSet;
026
027 import javax.servlet.ServletContext;
028 import javax.servlet.http.HttpServletRequest;
029 import javax.servlet.http.HttpServletResponse;
030 import javax.ws.rs.Consumes;
031 import javax.ws.rs.DefaultValue;
032 import javax.ws.rs.GET;
033 import javax.ws.rs.POST;
034 import javax.ws.rs.PUT;
035 import javax.ws.rs.Path;
036 import javax.ws.rs.PathParam;
037 import javax.ws.rs.Produces;
038 import javax.ws.rs.QueryParam;
039 import javax.ws.rs.core.Context;
040 import javax.ws.rs.core.MediaType;
041 import javax.ws.rs.core.Response;
042
043 import com.google.common.annotations.VisibleForTesting;
044 import org.apache.commons.logging.Log;
045 import org.apache.commons.logging.LogFactory;
046 import org.apache.hadoop.conf.Configuration;
047 import org.apache.hadoop.fs.CreateFlag;
048 import org.apache.hadoop.fs.FSDataOutputStream;
049 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
050 import org.apache.hadoop.fs.permission.FsPermission;
051 import org.apache.hadoop.hdfs.DFSClient;
052 import org.apache.hadoop.hdfs.HAUtil;
053 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
054 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
055 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
056 import org.apache.hadoop.hdfs.server.datanode.DataNode;
057 import org.apache.hadoop.hdfs.web.JsonUtil;
058 import org.apache.hadoop.hdfs.web.ParamFilter;
059 import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
060 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
061 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
062 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
063 import org.apache.hadoop.hdfs.web.resources.DelegationParam;
064 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
065 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
066 import org.apache.hadoop.hdfs.web.resources.LengthParam;
067 import org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
068 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
069 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
070 import org.apache.hadoop.hdfs.web.resources.Param;
071 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
072 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
073 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
074 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
075 import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
076 import org.apache.hadoop.io.IOUtils;
077 import org.apache.hadoop.io.Text;
078 import org.apache.hadoop.security.SecurityUtil;
079 import org.apache.hadoop.security.UserGroupInformation;
080 import org.apache.hadoop.security.token.Token;
081
082 import com.sun.jersey.spi.container.ResourceFilters;
083
084 /** Web-hdfs DataNode implementation. */
085 @Path("")
086 @ResourceFilters(ParamFilter.class)
087 public class DatanodeWebHdfsMethods {
088 public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
089
090 private static final UriFsPathParam ROOT = new UriFsPathParam("");
091
092 private @Context ServletContext context;
093 private @Context HttpServletRequest request;
094 private @Context HttpServletResponse response;
095
096 private void init(final UserGroupInformation ugi,
097 final DelegationParam delegation, final String nnId,
098 final UriFsPathParam path, final HttpOpParam<?> op,
099 final Param<?, ?>... parameters) throws IOException {
100 if (LOG.isTraceEnabled()) {
101 LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
102 + ", ugi=" + ugi + Param.toSortedString(", ", parameters));
103 }
104 if (nnId == null) {
105 throw new IllegalArgumentException(NamenodeAddressParam.NAME
106 + " is not specified.");
107 }
108
109 //clear content type
110 response.setContentType(null);
111
112 if (UserGroupInformation.isSecurityEnabled()) {
113 //add a token for RPC.
114 final Token<DelegationTokenIdentifier> token = deserializeToken
115 (delegation.getValue(), nnId);
116 ugi.addToken(token);
117 }
118 }
119
120 @VisibleForTesting
121 Token<DelegationTokenIdentifier> deserializeToken
122 (String delegation,String nnId) throws IOException {
123 final DataNode datanode = (DataNode) context.getAttribute("datanode");
124 final Configuration conf = datanode.getConf();
125 final Token<DelegationTokenIdentifier> token = new
126 Token<DelegationTokenIdentifier>();
127 token.decodeFromUrlString(delegation);
128 URI nnUri = URI.create(HdfsConstants.HDFS_URI_SCHEME +
129 "://" + nnId);
130 boolean isHA = HAUtil.isLogicalUri(conf, nnUri);
131 if (isHA) {
132 token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri));
133 } else {
134 token.setService(SecurityUtil.buildTokenService(nnUri));
135 }
136 token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
137 return token;
138 }
139
140 /** Handle HTTP PUT request for the root. */
141 @PUT
142 @Path("/")
143 @Consumes({"*/*"})
144 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
145 public Response putRoot(
146 final InputStream in,
147 @Context final UserGroupInformation ugi,
148 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
149 final DelegationParam delegation,
150 @QueryParam(NamenodeAddressParam.NAME)
151 @DefaultValue(NamenodeAddressParam.DEFAULT)
152 final NamenodeAddressParam namenode,
153 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
154 final PutOpParam op,
155 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
156 final PermissionParam permission,
157 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
158 final OverwriteParam overwrite,
159 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
160 final BufferSizeParam bufferSize,
161 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
162 final ReplicationParam replication,
163 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
164 final BlockSizeParam blockSize
165 ) throws IOException, InterruptedException {
166 return put(in, ugi, delegation, namenode, ROOT, op, permission,
167 overwrite, bufferSize, replication, blockSize);
168 }
169
170 /** Handle HTTP PUT request. */
171 @PUT
172 @Path("{" + UriFsPathParam.NAME + ":.*}")
173 @Consumes({"*/*"})
174 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
175 public Response put(
176 final InputStream in,
177 @Context final UserGroupInformation ugi,
178 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
179 final DelegationParam delegation,
180 @QueryParam(NamenodeAddressParam.NAME)
181 @DefaultValue(NamenodeAddressParam.DEFAULT)
182 final NamenodeAddressParam namenode,
183 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
184 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
185 final PutOpParam op,
186 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
187 final PermissionParam permission,
188 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
189 final OverwriteParam overwrite,
190 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
191 final BufferSizeParam bufferSize,
192 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
193 final ReplicationParam replication,
194 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
195 final BlockSizeParam blockSize
196 ) throws IOException, InterruptedException {
197
198 final String nnId = namenode.getValue();
199 init(ugi, delegation, nnId, path, op, permission,
200 overwrite, bufferSize, replication, blockSize);
201
202 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
203 @Override
204 public Response run() throws IOException, URISyntaxException {
205 return put(in, nnId, path.getAbsolutePath(), op,
206 permission, overwrite, bufferSize, replication, blockSize);
207 }
208 });
209 }
210
211 private Response put(
212 final InputStream in,
213 final String nnId,
214 final String fullpath,
215 final PutOpParam op,
216 final PermissionParam permission,
217 final OverwriteParam overwrite,
218 final BufferSizeParam bufferSize,
219 final ReplicationParam replication,
220 final BlockSizeParam blockSize
221 ) throws IOException, URISyntaxException {
222 final DataNode datanode = (DataNode)context.getAttribute("datanode");
223
224 switch(op.getValue()) {
225 case CREATE:
226 {
227 final Configuration conf = new Configuration(datanode.getConf());
228 conf.set(FsPermission.UMASK_LABEL, "000");
229
230 final int b = bufferSize.getValue(conf);
231 DFSClient dfsclient = newDfsClient(nnId, conf);
232 FSDataOutputStream out = null;
233 try {
234 out = new FSDataOutputStream(dfsclient.create(
235 fullpath, permission.getFsPermission(),
236 overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
237 : EnumSet.of(CreateFlag.CREATE),
238 replication.getValue(conf), blockSize.getValue(conf), null, b, null), null);
239 IOUtils.copyBytes(in, out, b);
240 out.close();
241 out = null;
242 dfsclient.close();
243 dfsclient = null;
244 } finally {
245 IOUtils.cleanup(LOG, out);
246 IOUtils.cleanup(LOG, dfsclient);
247 }
248 final String scheme = "http".equals(request.getScheme()) ?
249 WebHdfsFileSystem.SCHEME : SWebHdfsFileSystem.SCHEME;
250 final URI uri = new URI(scheme, nnId, fullpath, null, null);
251 return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
252 }
253 default:
254 throw new UnsupportedOperationException(op + " is not supported");
255 }
256 }
257
258 /** Handle HTTP POST request for the root for the root. */
259 @POST
260 @Path("/")
261 @Consumes({"*/*"})
262 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
263 public Response postRoot(
264 final InputStream in,
265 @Context final UserGroupInformation ugi,
266 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
267 final DelegationParam delegation,
268 @QueryParam(NamenodeAddressParam.NAME)
269 @DefaultValue(NamenodeAddressParam.DEFAULT)
270 final NamenodeAddressParam namenode,
271 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
272 final PostOpParam op,
273 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
274 final BufferSizeParam bufferSize
275 ) throws IOException, InterruptedException {
276 return post(in, ugi, delegation, namenode, ROOT, op, bufferSize);
277 }
278
279 /** Handle HTTP POST request. */
280 @POST
281 @Path("{" + UriFsPathParam.NAME + ":.*}")
282 @Consumes({"*/*"})
283 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
284 public Response post(
285 final InputStream in,
286 @Context final UserGroupInformation ugi,
287 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
288 final DelegationParam delegation,
289 @QueryParam(NamenodeAddressParam.NAME)
290 @DefaultValue(NamenodeAddressParam.DEFAULT)
291 final NamenodeAddressParam namenode,
292 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
293 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
294 final PostOpParam op,
295 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
296 final BufferSizeParam bufferSize
297 ) throws IOException, InterruptedException {
298
299 final String nnId = namenode.getValue();
300 init(ugi, delegation, nnId, path, op, bufferSize);
301
302 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
303 @Override
304 public Response run() throws IOException {
305 return post(in, nnId, path.getAbsolutePath(), op,
306 bufferSize);
307 }
308 });
309 }
310
311 private Response post(
312 final InputStream in,
313 final String nnId,
314 final String fullpath,
315 final PostOpParam op,
316 final BufferSizeParam bufferSize
317 ) throws IOException {
318 final DataNode datanode = (DataNode)context.getAttribute("datanode");
319
320 switch(op.getValue()) {
321 case APPEND:
322 {
323 final Configuration conf = new Configuration(datanode.getConf());
324 final int b = bufferSize.getValue(conf);
325 DFSClient dfsclient = newDfsClient(nnId, conf);
326 FSDataOutputStream out = null;
327 try {
328 out = dfsclient.append(fullpath, b, null, null);
329 IOUtils.copyBytes(in, out, b);
330 out.close();
331 out = null;
332 dfsclient.close();
333 dfsclient = null;
334 } finally {
335 IOUtils.cleanup(LOG, out);
336 IOUtils.cleanup(LOG, dfsclient);
337 }
338 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
339 }
340 default:
341 throw new UnsupportedOperationException(op + " is not supported");
342 }
343 }
344
345 /** Handle HTTP GET request for the root. */
346 @GET
347 @Path("/")
348 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
349 public Response getRoot(
350 @Context final UserGroupInformation ugi,
351 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
352 final DelegationParam delegation,
353 @QueryParam(NamenodeAddressParam.NAME)
354 @DefaultValue(NamenodeAddressParam.DEFAULT)
355 final NamenodeAddressParam namenode,
356 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
357 final GetOpParam op,
358 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
359 final OffsetParam offset,
360 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
361 final LengthParam length,
362 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
363 final BufferSizeParam bufferSize
364 ) throws IOException, InterruptedException {
365 return get(ugi, delegation, namenode, ROOT, op, offset, length,
366 bufferSize);
367 }
368
369 /** Handle HTTP GET request. */
370 @GET
371 @Path("{" + UriFsPathParam.NAME + ":.*}")
372 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
373 public Response get(
374 @Context final UserGroupInformation ugi,
375 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
376 final DelegationParam delegation,
377 @QueryParam(NamenodeAddressParam.NAME)
378 @DefaultValue(NamenodeAddressParam.DEFAULT)
379 final NamenodeAddressParam namenode,
380 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
381 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
382 final GetOpParam op,
383 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
384 final OffsetParam offset,
385 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
386 final LengthParam length,
387 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
388 final BufferSizeParam bufferSize
389 ) throws IOException, InterruptedException {
390
391 final String nnId = namenode.getValue();
392 init(ugi, delegation, nnId, path, op, offset, length, bufferSize);
393
394 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
395 @Override
396 public Response run() throws IOException {
397 return get(nnId, path.getAbsolutePath(), op, offset,
398 length, bufferSize);
399 }
400 });
401 }
402
403 private Response get(
404 final String nnId,
405 final String fullpath,
406 final GetOpParam op,
407 final OffsetParam offset,
408 final LengthParam length,
409 final BufferSizeParam bufferSize
410 ) throws IOException {
411 final DataNode datanode = (DataNode)context.getAttribute("datanode");
412 final Configuration conf = new Configuration(datanode.getConf());
413
414 switch(op.getValue()) {
415 case OPEN:
416 {
417 final int b = bufferSize.getValue(conf);
418 final DFSClient dfsclient = newDfsClient(nnId, conf);
419 HdfsDataInputStream in = null;
420 try {
421 in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
422 in.seek(offset.getValue());
423 } catch(IOException ioe) {
424 IOUtils.cleanup(LOG, in);
425 IOUtils.cleanup(LOG, dfsclient);
426 throw ioe;
427 }
428
429 final long n = length.getValue() != null ?
430 Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
431 in.getVisibleLength() - offset.getValue();
432
433 /**
434 * Allow the Web UI to perform an AJAX request to get the data.
435 */
436 return Response.ok(new OpenEntity(in, n, dfsclient))
437 .type(MediaType.APPLICATION_OCTET_STREAM)
438 .header("Access-Control-Allow-Methods", "GET")
439 .header("Access-Control-Allow-Origin", "*")
440 .build();
441 }
442 case GETFILECHECKSUM:
443 {
444 MD5MD5CRC32FileChecksum checksum = null;
445 DFSClient dfsclient = newDfsClient(nnId, conf);
446 try {
447 checksum = dfsclient.getFileChecksum(fullpath);
448 dfsclient.close();
449 dfsclient = null;
450 } finally {
451 IOUtils.cleanup(LOG, dfsclient);
452 }
453 final String js = JsonUtil.toJsonString(checksum);
454 return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
455 }
456 default:
457 throw new UnsupportedOperationException(op + " is not supported");
458 }
459 }
460
461 private static DFSClient newDfsClient(String nnId,
462 Configuration conf) throws IOException {
463 URI uri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://" + nnId);
464 return new DFSClient(uri, conf);
465 }
466 }