001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.datanode.web.resources;
019
020 import java.io.IOException;
021 import java.io.InputStream;
022 import java.net.InetSocketAddress;
023 import java.net.URI;
024 import java.net.URISyntaxException;
025 import java.security.PrivilegedExceptionAction;
026 import java.util.EnumSet;
027
028 import javax.servlet.ServletContext;
029 import javax.servlet.http.HttpServletResponse;
030 import javax.ws.rs.Consumes;
031 import javax.ws.rs.DefaultValue;
032 import javax.ws.rs.GET;
033 import javax.ws.rs.POST;
034 import javax.ws.rs.PUT;
035 import javax.ws.rs.Path;
036 import javax.ws.rs.PathParam;
037 import javax.ws.rs.Produces;
038 import javax.ws.rs.QueryParam;
039 import javax.ws.rs.core.Context;
040 import javax.ws.rs.core.MediaType;
041 import javax.ws.rs.core.Response;
042
043 import org.apache.commons.logging.Log;
044 import org.apache.commons.logging.LogFactory;
045 import org.apache.hadoop.conf.Configuration;
046 import org.apache.hadoop.fs.CreateFlag;
047 import org.apache.hadoop.fs.FSDataOutputStream;
048 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
049 import org.apache.hadoop.fs.permission.FsPermission;
050 import org.apache.hadoop.hdfs.DFSClient;
051 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
052 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
053 import org.apache.hadoop.hdfs.server.datanode.DataNode;
054 import org.apache.hadoop.hdfs.server.namenode.NameNode;
055 import org.apache.hadoop.hdfs.web.JsonUtil;
056 import org.apache.hadoop.hdfs.web.ParamFilter;
057 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
058 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
059 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
060 import org.apache.hadoop.hdfs.web.resources.DelegationParam;
061 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
062 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
063 import org.apache.hadoop.hdfs.web.resources.LengthParam;
064 import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam;
065 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
066 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
067 import org.apache.hadoop.hdfs.web.resources.Param;
068 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
069 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
070 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
071 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
072 import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
073 import org.apache.hadoop.io.IOUtils;
074 import org.apache.hadoop.security.SecurityUtil;
075 import org.apache.hadoop.security.UserGroupInformation;
076 import org.apache.hadoop.security.token.Token;
077
078 import com.sun.jersey.spi.container.ResourceFilters;
079
080 /** Web-hdfs DataNode implementation. */
081 @Path("")
082 @ResourceFilters(ParamFilter.class)
083 public class DatanodeWebHdfsMethods {
084 public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
085
086 private static final UriFsPathParam ROOT = new UriFsPathParam("");
087
088 private @Context ServletContext context;
089 private @Context HttpServletResponse response;
090
091 private void init(final UserGroupInformation ugi,
092 final DelegationParam delegation, final InetSocketAddress nnRpcAddr,
093 final UriFsPathParam path, final HttpOpParam<?> op,
094 final Param<?, ?>... parameters) throws IOException {
095 if (LOG.isTraceEnabled()) {
096 LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
097 + ", ugi=" + ugi + Param.toSortedString(", ", parameters));
098 }
099 if (nnRpcAddr == null) {
100 throw new IllegalArgumentException(NamenodeRpcAddressParam.NAME
101 + " is not specified.");
102 }
103
104 //clear content type
105 response.setContentType(null);
106
107 if (UserGroupInformation.isSecurityEnabled()) {
108 //add a token for RPC.
109 final Token<DelegationTokenIdentifier> token =
110 new Token<DelegationTokenIdentifier>();
111 token.decodeFromUrlString(delegation.getValue());
112 SecurityUtil.setTokenService(token, nnRpcAddr);
113 token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
114 ugi.addToken(token);
115 }
116 }
117
118 /** Handle HTTP PUT request for the root. */
119 @PUT
120 @Path("/")
121 @Consumes({"*/*"})
122 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
123 public Response putRoot(
124 final InputStream in,
125 @Context final UserGroupInformation ugi,
126 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
127 final DelegationParam delegation,
128 @QueryParam(NamenodeRpcAddressParam.NAME)
129 @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
130 final NamenodeRpcAddressParam namenodeRpcAddress,
131 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
132 final PutOpParam op,
133 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
134 final PermissionParam permission,
135 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
136 final OverwriteParam overwrite,
137 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
138 final BufferSizeParam bufferSize,
139 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
140 final ReplicationParam replication,
141 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
142 final BlockSizeParam blockSize
143 ) throws IOException, InterruptedException {
144 return put(in, ugi, delegation, namenodeRpcAddress, ROOT, op, permission,
145 overwrite, bufferSize, replication, blockSize);
146 }
147
148 /** Handle HTTP PUT request. */
149 @PUT
150 @Path("{" + UriFsPathParam.NAME + ":.*}")
151 @Consumes({"*/*"})
152 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
153 public Response put(
154 final InputStream in,
155 @Context final UserGroupInformation ugi,
156 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
157 final DelegationParam delegation,
158 @QueryParam(NamenodeRpcAddressParam.NAME)
159 @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
160 final NamenodeRpcAddressParam namenodeRpcAddress,
161 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
162 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
163 final PutOpParam op,
164 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
165 final PermissionParam permission,
166 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
167 final OverwriteParam overwrite,
168 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
169 final BufferSizeParam bufferSize,
170 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
171 final ReplicationParam replication,
172 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
173 final BlockSizeParam blockSize
174 ) throws IOException, InterruptedException {
175
176 final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
177 init(ugi, delegation, nnRpcAddr, path, op, permission,
178 overwrite, bufferSize, replication, blockSize);
179
180 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
181 @Override
182 public Response run() throws IOException, URISyntaxException {
183 return put(in, ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op,
184 permission, overwrite, bufferSize, replication, blockSize);
185 }
186 });
187 }
188
189 private Response put(
190 final InputStream in,
191 final UserGroupInformation ugi,
192 final DelegationParam delegation,
193 final InetSocketAddress nnRpcAddr,
194 final String fullpath,
195 final PutOpParam op,
196 final PermissionParam permission,
197 final OverwriteParam overwrite,
198 final BufferSizeParam bufferSize,
199 final ReplicationParam replication,
200 final BlockSizeParam blockSize
201 ) throws IOException, URISyntaxException {
202 final DataNode datanode = (DataNode)context.getAttribute("datanode");
203
204 switch(op.getValue()) {
205 case CREATE:
206 {
207 final Configuration conf = new Configuration(datanode.getConf());
208 conf.set(FsPermission.UMASK_LABEL, "000");
209
210 final int b = bufferSize.getValue(conf);
211 DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
212 FSDataOutputStream out = null;
213 try {
214 out = new FSDataOutputStream(dfsclient.create(
215 fullpath, permission.getFsPermission(),
216 overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
217 : EnumSet.of(CreateFlag.CREATE),
218 replication.getValue(conf), blockSize.getValue(conf), null, b, null), null);
219 IOUtils.copyBytes(in, out, b);
220 out.close();
221 out = null;
222 dfsclient.close();
223 dfsclient = null;
224 } finally {
225 IOUtils.cleanup(LOG, out);
226 IOUtils.cleanup(LOG, dfsclient);
227 }
228 final InetSocketAddress nnHttpAddr = NameNode.getHttpAddress(conf);
229 final URI uri = new URI(WebHdfsFileSystem.SCHEME, null,
230 nnHttpAddr.getHostName(), nnHttpAddr.getPort(), fullpath, null, null);
231 return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
232 }
233 default:
234 throw new UnsupportedOperationException(op + " is not supported");
235 }
236 }
237
238 /** Handle HTTP POST request for the root for the root. */
239 @POST
240 @Path("/")
241 @Consumes({"*/*"})
242 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
243 public Response postRoot(
244 final InputStream in,
245 @Context final UserGroupInformation ugi,
246 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
247 final DelegationParam delegation,
248 @QueryParam(NamenodeRpcAddressParam.NAME)
249 @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
250 final NamenodeRpcAddressParam namenodeRpcAddress,
251 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
252 final PostOpParam op,
253 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
254 final BufferSizeParam bufferSize
255 ) throws IOException, InterruptedException {
256 return post(in, ugi, delegation, namenodeRpcAddress, ROOT, op, bufferSize);
257 }
258
259 /** Handle HTTP POST request. */
260 @POST
261 @Path("{" + UriFsPathParam.NAME + ":.*}")
262 @Consumes({"*/*"})
263 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
264 public Response post(
265 final InputStream in,
266 @Context final UserGroupInformation ugi,
267 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
268 final DelegationParam delegation,
269 @QueryParam(NamenodeRpcAddressParam.NAME)
270 @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
271 final NamenodeRpcAddressParam namenodeRpcAddress,
272 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
273 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
274 final PostOpParam op,
275 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
276 final BufferSizeParam bufferSize
277 ) throws IOException, InterruptedException {
278
279 final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
280 init(ugi, delegation, nnRpcAddr, path, op, bufferSize);
281
282 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
283 @Override
284 public Response run() throws IOException {
285 return post(in, ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op,
286 bufferSize);
287 }
288 });
289 }
290
291 private Response post(
292 final InputStream in,
293 final UserGroupInformation ugi,
294 final DelegationParam delegation,
295 final InetSocketAddress nnRpcAddr,
296 final String fullpath,
297 final PostOpParam op,
298 final BufferSizeParam bufferSize
299 ) throws IOException {
300 final DataNode datanode = (DataNode)context.getAttribute("datanode");
301
302 switch(op.getValue()) {
303 case APPEND:
304 {
305 final Configuration conf = new Configuration(datanode.getConf());
306 final int b = bufferSize.getValue(conf);
307 DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
308 FSDataOutputStream out = null;
309 try {
310 out = dfsclient.append(fullpath, b, null, null);
311 IOUtils.copyBytes(in, out, b);
312 out.close();
313 out = null;
314 dfsclient.close();
315 dfsclient = null;
316 } finally {
317 IOUtils.cleanup(LOG, out);
318 IOUtils.cleanup(LOG, dfsclient);
319 }
320 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
321 }
322 default:
323 throw new UnsupportedOperationException(op + " is not supported");
324 }
325 }
326
327 /** Handle HTTP GET request for the root. */
328 @GET
329 @Path("/")
330 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
331 public Response getRoot(
332 @Context final UserGroupInformation ugi,
333 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
334 final DelegationParam delegation,
335 @QueryParam(NamenodeRpcAddressParam.NAME)
336 @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
337 final NamenodeRpcAddressParam namenodeRpcAddress,
338 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
339 final GetOpParam op,
340 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
341 final OffsetParam offset,
342 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
343 final LengthParam length,
344 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
345 final BufferSizeParam bufferSize
346 ) throws IOException, InterruptedException {
347 return get(ugi, delegation, namenodeRpcAddress, ROOT, op, offset, length,
348 bufferSize);
349 }
350
351 /** Handle HTTP GET request. */
352 @GET
353 @Path("{" + UriFsPathParam.NAME + ":.*}")
354 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
355 public Response get(
356 @Context final UserGroupInformation ugi,
357 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
358 final DelegationParam delegation,
359 @QueryParam(NamenodeRpcAddressParam.NAME)
360 @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
361 final NamenodeRpcAddressParam namenodeRpcAddress,
362 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
363 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
364 final GetOpParam op,
365 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
366 final OffsetParam offset,
367 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
368 final LengthParam length,
369 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
370 final BufferSizeParam bufferSize
371 ) throws IOException, InterruptedException {
372
373 final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
374 init(ugi, delegation, nnRpcAddr, path, op, offset, length, bufferSize);
375
376 return ugi.doAs(new PrivilegedExceptionAction<Response>() {
377 @Override
378 public Response run() throws IOException {
379 return get(ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op,
380 offset, length, bufferSize);
381 }
382 });
383 }
384
385 private Response get(
386 final UserGroupInformation ugi,
387 final DelegationParam delegation,
388 final InetSocketAddress nnRpcAddr,
389 final String fullpath,
390 final GetOpParam op,
391 final OffsetParam offset,
392 final LengthParam length,
393 final BufferSizeParam bufferSize
394 ) throws IOException {
395 final DataNode datanode = (DataNode)context.getAttribute("datanode");
396 final Configuration conf = new Configuration(datanode.getConf());
397
398 switch(op.getValue()) {
399 case OPEN:
400 {
401 final int b = bufferSize.getValue(conf);
402 final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
403 HdfsDataInputStream in = null;
404 try {
405 in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
406 in.seek(offset.getValue());
407 } catch(IOException ioe) {
408 IOUtils.cleanup(LOG, in);
409 IOUtils.cleanup(LOG, dfsclient);
410 throw ioe;
411 }
412
413 final long n = length.getValue() != null ?
414 Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
415 in.getVisibleLength() - offset.getValue();
416 return Response.ok(new OpenEntity(in, n, dfsclient)).type(
417 MediaType.APPLICATION_OCTET_STREAM).build();
418 }
419 case GETFILECHECKSUM:
420 {
421 MD5MD5CRC32FileChecksum checksum = null;
422 DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
423 try {
424 checksum = dfsclient.getFileChecksum(fullpath);
425 dfsclient.close();
426 dfsclient = null;
427 } finally {
428 IOUtils.cleanup(LOG, dfsclient);
429 }
430 final String js = JsonUtil.toJsonString(checksum);
431 return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
432 }
433 default:
434 throw new UnsupportedOperationException(op + " is not supported");
435 }
436 }
437 }