001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.namenode;
019
020 import static org.apache.hadoop.util.Time.now;
021
022 import java.io.Closeable;
023 import java.io.FileNotFoundException;
024 import java.io.IOException;
025 import java.util.ArrayList;
026 import java.util.Arrays;
027 import java.util.List;
028 import java.util.concurrent.TimeUnit;
029 import java.util.concurrent.locks.Condition;
030 import java.util.concurrent.locks.ReentrantReadWriteLock;
031
032 import org.apache.hadoop.HadoopIllegalArgumentException;
033 import org.apache.hadoop.conf.Configuration;
034 import org.apache.hadoop.fs.ContentSummary;
035 import org.apache.hadoop.fs.FileAlreadyExistsException;
036 import org.apache.hadoop.fs.Options;
037 import org.apache.hadoop.fs.Options.Rename;
038 import org.apache.hadoop.fs.ParentNotDirectoryException;
039 import org.apache.hadoop.fs.Path;
040 import org.apache.hadoop.fs.PathIsNotDirectoryException;
041 import org.apache.hadoop.fs.UnresolvedLinkException;
042 import org.apache.hadoop.fs.permission.FsAction;
043 import org.apache.hadoop.fs.permission.FsPermission;
044 import org.apache.hadoop.fs.permission.PermissionStatus;
045 import org.apache.hadoop.hdfs.DFSConfigKeys;
046 import org.apache.hadoop.hdfs.DFSUtil;
047 import org.apache.hadoop.hdfs.DistributedFileSystem;
048 import org.apache.hadoop.hdfs.protocol.Block;
049 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
050 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
051 import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
052 import org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
053 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
054 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
055 import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
056 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
057 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
058 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
059 import org.apache.hadoop.hdfs.protocol.SnapshotException;
060 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
061 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
062 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
063 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
064 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
065 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
066 import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
067 import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
068 import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot;
069 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
070 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
071 import org.apache.hadoop.hdfs.util.ByteArray;
072 import org.apache.hadoop.hdfs.util.ReadOnlyList;
073
074 import com.google.common.annotations.VisibleForTesting;
075 import com.google.common.base.Preconditions;
076
077 /*************************************************
078 * FSDirectory stores the filesystem directory state.
079 * It handles writing/loading values to disk, and logging
080 * changes as we go.
081 *
082 * It keeps the filename->blockset mapping always-current
083 * and logged to disk.
084 *
085 *************************************************/
086 public class FSDirectory implements Closeable {
087 private static INodeDirectoryWithQuota createRoot(FSNamesystem namesystem) {
088 final INodeDirectoryWithQuota r = new INodeDirectoryWithQuota(
089 INodeId.ROOT_INODE_ID,
090 INodeDirectory.ROOT_NAME,
091 namesystem.createFsOwnerPermissions(new FsPermission((short) 0755)));
092 final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(r);
093 s.setSnapshotQuota(0);
094 return s;
095 }
096
097 @VisibleForTesting
098 static boolean CHECK_RESERVED_FILE_NAMES = true;
099 public final static String DOT_RESERVED_STRING = ".reserved";
100 public final static String DOT_RESERVED_PATH_PREFIX = Path.SEPARATOR
101 + DOT_RESERVED_STRING;
102 public final static byte[] DOT_RESERVED =
103 DFSUtil.string2Bytes(DOT_RESERVED_STRING);
104 public final static String DOT_INODES_STRING = ".inodes";
105 public final static byte[] DOT_INODES =
106 DFSUtil.string2Bytes(DOT_INODES_STRING);
107 INodeDirectoryWithQuota rootDir;
108 FSImage fsImage;
109 private final FSNamesystem namesystem;
110 private volatile boolean ready = false;
111 private final int maxComponentLength;
112 private final int maxDirItems;
113 private final int lsLimit; // max list limit
114 private final INodeMap inodeMap; // Synchronized by dirLock
115
116 // lock to protect the directory and BlockMap
117 private ReentrantReadWriteLock dirLock;
118 private Condition cond;
119
120 // utility methods to acquire and release read lock and write lock
121 void readLock() {
122 this.dirLock.readLock().lock();
123 }
124
125 void readUnlock() {
126 this.dirLock.readLock().unlock();
127 }
128
129 void writeLock() {
130 this.dirLock.writeLock().lock();
131 }
132
133 void writeUnlock() {
134 this.dirLock.writeLock().unlock();
135 }
136
137 boolean hasWriteLock() {
138 return this.dirLock.isWriteLockedByCurrentThread();
139 }
140
141 boolean hasReadLock() {
142 return this.dirLock.getReadHoldCount() > 0;
143 }
144
145 /**
146 * Caches frequently used file names used in {@link INode} to reuse
147 * byte[] objects and reduce heap usage.
148 */
149 private final NameCache<ByteArray> nameCache;
150
151 FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
152 this.dirLock = new ReentrantReadWriteLock(true); // fair
153 this.cond = dirLock.writeLock().newCondition();
154 rootDir = createRoot(ns);
155 inodeMap = INodeMap.newInstance(rootDir);
156 this.fsImage = fsImage;
157 int configuredLimit = conf.getInt(
158 DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
159 this.lsLimit = configuredLimit>0 ?
160 configuredLimit : DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT;
161
162 // filesystem limits
163 this.maxComponentLength = conf.getInt(
164 DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY,
165 DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT);
166 this.maxDirItems = conf.getInt(
167 DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY,
168 DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT);
169
170 int threshold = conf.getInt(
171 DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY,
172 DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT);
173 NameNode.LOG.info("Caching file names occuring more than " + threshold
174 + " times");
175 nameCache = new NameCache<ByteArray>(threshold);
176 namesystem = ns;
177 }
178
179 private FSNamesystem getFSNamesystem() {
180 return namesystem;
181 }
182
183 private BlockManager getBlockManager() {
184 return getFSNamesystem().getBlockManager();
185 }
186
187 /** @return the root directory inode. */
188 public INodeDirectoryWithQuota getRoot() {
189 return rootDir;
190 }
191
192 /**
193 * Notify that loading of this FSDirectory is complete, and
194 * it is ready for use
195 */
196 void imageLoadComplete() {
197 Preconditions.checkState(!ready, "FSDirectory already loaded");
198 setReady();
199 }
200
201 void setReady() {
202 if(ready) return;
203 writeLock();
204 try {
205 setReady(true);
206 this.nameCache.initialized();
207 cond.signalAll();
208 } finally {
209 writeUnlock();
210 }
211 }
212
213 //This is for testing purposes only
214 @VisibleForTesting
215 boolean isReady() {
216 return ready;
217 }
218
219 // exposed for unit tests
220 protected void setReady(boolean flag) {
221 ready = flag;
222 }
223
224 private void incrDeletedFileCount(long count) {
225 if (getFSNamesystem() != null)
226 NameNode.getNameNodeMetrics().incrFilesDeleted(count);
227 }
228
229 /**
230 * Shutdown the filestore
231 */
232 @Override
233 public void close() throws IOException {
234 fsImage.close();
235 }
236
237 /**
238 * Block until the object is ready to be used.
239 */
240 void waitForReady() {
241 if (!ready) {
242 writeLock();
243 try {
244 while (!ready) {
245 try {
246 cond.await(5000, TimeUnit.MILLISECONDS);
247 } catch (InterruptedException ie) {
248 }
249 }
250 } finally {
251 writeUnlock();
252 }
253 }
254 }
255
256 /**
257 * Add the given filename to the fs.
258 * @throws FileAlreadyExistsException
259 * @throws QuotaExceededException
260 * @throws UnresolvedLinkException
261 * @throws SnapshotAccessControlException
262 */
263 INodeFileUnderConstruction addFile(String path,
264 PermissionStatus permissions,
265 short replication,
266 long preferredBlockSize,
267 String clientName,
268 String clientMachine,
269 DatanodeDescriptor clientNode)
270 throws FileAlreadyExistsException, QuotaExceededException,
271 UnresolvedLinkException, SnapshotAccessControlException {
272 waitForReady();
273
274 // Always do an implicit mkdirs for parent directory tree.
275 long modTime = now();
276
277 Path parent = new Path(path).getParent();
278 if (parent == null) {
279 // Trying to add "/" as a file - this path has no
280 // parent -- avoids an NPE below.
281 return null;
282 }
283
284 if (!mkdirs(parent.toString(), permissions, true, modTime)) {
285 return null;
286 }
287 INodeFileUnderConstruction newNode = new INodeFileUnderConstruction(
288 namesystem.allocateNewInodeId(),
289 permissions,replication,
290 preferredBlockSize, modTime, clientName,
291 clientMachine, clientNode);
292 boolean added = false;
293 writeLock();
294 try {
295 added = addINode(path, newNode);
296 } finally {
297 writeUnlock();
298 }
299 if (!added) {
300 NameNode.stateChangeLog.info("DIR* addFile: failed to add " + path);
301 return null;
302 }
303
304 if(NameNode.stateChangeLog.isDebugEnabled()) {
305 NameNode.stateChangeLog.debug("DIR* addFile: " + path + " is added");
306 }
307 return newNode;
308 }
309
310 INodeFile unprotectedAddFile( long id,
311 String path,
312 PermissionStatus permissions,
313 short replication,
314 long modificationTime,
315 long atime,
316 long preferredBlockSize,
317 boolean underConstruction,
318 String clientName,
319 String clientMachine) {
320 final INodeFile newNode;
321 assert hasWriteLock();
322 if (underConstruction) {
323 newNode = new INodeFileUnderConstruction(id, permissions, replication,
324 preferredBlockSize, modificationTime, clientName, clientMachine, null);
325 } else {
326 newNode = new INodeFile(id, null, permissions, modificationTime, atime,
327 BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize);
328 }
329
330 try {
331 if (addINode(path, newNode)) {
332 return newNode;
333 }
334 } catch (IOException e) {
335 if(NameNode.stateChangeLog.isDebugEnabled()) {
336 NameNode.stateChangeLog.debug(
337 "DIR* FSDirectory.unprotectedAddFile: exception when add " + path
338 + " to the file system", e);
339 }
340 }
341 return null;
342 }
343
344 /**
345 * Add a block to the file. Returns a reference to the added block.
346 */
347 BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
348 DatanodeDescriptor targets[]) throws IOException {
349 waitForReady();
350
351 writeLock();
352 try {
353 final INodeFileUnderConstruction fileINode =
354 INodeFileUnderConstruction.valueOf(inodesInPath.getLastINode(), path);
355
356 // check quota limits and updated space consumed
357 updateCount(inodesInPath, 0, fileINode.getBlockDiskspace(), true);
358
359 // associate new last block for the file
360 BlockInfoUnderConstruction blockInfo =
361 new BlockInfoUnderConstruction(
362 block,
363 fileINode.getFileReplication(),
364 BlockUCState.UNDER_CONSTRUCTION,
365 targets);
366 getBlockManager().addBlockCollection(blockInfo, fileINode);
367 fileINode.addBlock(blockInfo);
368
369 if(NameNode.stateChangeLog.isDebugEnabled()) {
370 NameNode.stateChangeLog.debug("DIR* FSDirectory.addBlock: "
371 + path + " with " + block
372 + " block is added to the in-memory "
373 + "file system");
374 }
375 return blockInfo;
376 } finally {
377 writeUnlock();
378 }
379 }
380
381 /**
382 * Persist the block list for the inode.
383 */
384 void persistBlocks(String path, INodeFileUnderConstruction file,
385 boolean logRetryCache) {
386 waitForReady();
387
388 writeLock();
389 try {
390 fsImage.getEditLog().logUpdateBlocks(path, file, logRetryCache);
391 if(NameNode.stateChangeLog.isDebugEnabled()) {
392 NameNode.stateChangeLog.debug("DIR* FSDirectory.persistBlocks: "
393 +path+" with "+ file.getBlocks().length
394 +" blocks is persisted to the file system");
395 }
396 } finally {
397 writeUnlock();
398 }
399 }
400
401 /**
402 * Close file.
403 */
404 void closeFile(String path, INodeFile file) {
405 waitForReady();
406 writeLock();
407 try {
408 // file is closed
409 fsImage.getEditLog().logCloseFile(path, file);
410 if (NameNode.stateChangeLog.isDebugEnabled()) {
411 NameNode.stateChangeLog.debug("DIR* FSDirectory.closeFile: "
412 +path+" with "+ file.getBlocks().length
413 +" blocks is persisted to the file system");
414 }
415 } finally {
416 writeUnlock();
417 }
418 }
419
420 /**
421 * Remove a block from the file.
422 * @return Whether the block exists in the corresponding file
423 */
424 boolean removeBlock(String path, INodeFileUnderConstruction fileNode,
425 Block block) throws IOException {
426 waitForReady();
427
428 writeLock();
429 try {
430 return unprotectedRemoveBlock(path, fileNode, block);
431 } finally {
432 writeUnlock();
433 }
434 }
435
436 boolean unprotectedRemoveBlock(String path,
437 INodeFileUnderConstruction fileNode, Block block) throws IOException {
438 // modify file-> block and blocksMap
439 boolean removed = fileNode.removeLastBlock(block);
440 if (!removed) {
441 return false;
442 }
443 getBlockManager().removeBlockFromMap(block);
444
445 if(NameNode.stateChangeLog.isDebugEnabled()) {
446 NameNode.stateChangeLog.debug("DIR* FSDirectory.removeBlock: "
447 +path+" with "+block
448 +" block is removed from the file system");
449 }
450
451 // update space consumed
452 final INodesInPath iip = rootDir.getINodesInPath4Write(path, true);
453 updateCount(iip, 0, -fileNode.getBlockDiskspace(), true);
454 return true;
455 }
456
457 /**
458 * @throws SnapshotAccessControlException
459 * @see #unprotectedRenameTo(String, String, long)
460 * @deprecated Use {@link #renameTo(String, String, Rename...)} instead.
461 */
462 @Deprecated
463 boolean renameTo(String src, String dst, boolean logRetryCache)
464 throws QuotaExceededException, UnresolvedLinkException,
465 FileAlreadyExistsException, SnapshotAccessControlException, IOException {
466 if (NameNode.stateChangeLog.isDebugEnabled()) {
467 NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: "
468 +src+" to "+dst);
469 }
470 waitForReady();
471 long now = now();
472 writeLock();
473 try {
474 if (!unprotectedRenameTo(src, dst, now))
475 return false;
476 } finally {
477 writeUnlock();
478 }
479 fsImage.getEditLog().logRename(src, dst, now, logRetryCache);
480 return true;
481 }
482
483 /**
484 * @see #unprotectedRenameTo(String, String, long, Options.Rename...)
485 */
486 void renameTo(String src, String dst, boolean logRetryCache,
487 Options.Rename... options)
488 throws FileAlreadyExistsException, FileNotFoundException,
489 ParentNotDirectoryException, QuotaExceededException,
490 UnresolvedLinkException, IOException {
491 if (NameNode.stateChangeLog.isDebugEnabled()) {
492 NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src
493 + " to " + dst);
494 }
495 waitForReady();
496 long now = now();
497 writeLock();
498 try {
499 if (unprotectedRenameTo(src, dst, now, options)) {
500 incrDeletedFileCount(1);
501 }
502 } finally {
503 writeUnlock();
504 }
505 fsImage.getEditLog().logRename(src, dst, now, logRetryCache, options);
506 }
507
508 /**
509 * Change a path name
510 *
511 * @param src source path
512 * @param dst destination path
513 * @return true if rename succeeds; false otherwise
514 * @throws QuotaExceededException if the operation violates any quota limit
515 * @throws FileAlreadyExistsException if the src is a symlink that points to dst
516 * @throws SnapshotAccessControlException if path is in RO snapshot
517 * @deprecated See {@link #renameTo(String, String)}
518 */
519 @Deprecated
520 boolean unprotectedRenameTo(String src, String dst, long timestamp)
521 throws QuotaExceededException, UnresolvedLinkException,
522 FileAlreadyExistsException, SnapshotAccessControlException, IOException {
523 assert hasWriteLock();
524 INodesInPath srcIIP = rootDir.getINodesInPath4Write(src, false);
525 final INode srcInode = srcIIP.getLastINode();
526
527 // check the validation of the source
528 if (srcInode == null) {
529 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
530 + "failed to rename " + src + " to " + dst
531 + " because source does not exist");
532 return false;
533 }
534 if (srcIIP.getINodes().length == 1) {
535 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
536 +"failed to rename "+src+" to "+dst+ " because source is the root");
537 return false;
538 }
539
540 // srcInode and its subtree cannot contain snapshottable directories with
541 // snapshots
542 List<INodeDirectorySnapshottable> snapshottableDirs =
543 new ArrayList<INodeDirectorySnapshottable>();
544 checkSnapshot(srcInode, snapshottableDirs);
545
546 if (isDir(dst)) {
547 dst += Path.SEPARATOR + new Path(src).getName();
548 }
549
550 // check the validity of the destination
551 if (dst.equals(src)) {
552 return true;
553 }
554 if (srcInode.isSymlink() &&
555 dst.equals(srcInode.asSymlink().getSymlinkString())) {
556 throw new FileAlreadyExistsException(
557 "Cannot rename symlink "+src+" to its target "+dst);
558 }
559
560 // dst cannot be directory or a file under src
561 if (dst.startsWith(src) &&
562 dst.charAt(src.length()) == Path.SEPARATOR_CHAR) {
563 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
564 + "failed to rename " + src + " to " + dst
565 + " because destination starts with src");
566 return false;
567 }
568
569 byte[][] dstComponents = INode.getPathComponents(dst);
570 INodesInPath dstIIP = getExistingPathINodes(dstComponents);
571 if (dstIIP.isSnapshot()) {
572 throw new SnapshotAccessControlException(
573 "Modification on RO snapshot is disallowed");
574 }
575 if (dstIIP.getLastINode() != null) {
576 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
577 +"failed to rename "+src+" to "+dst+
578 " because destination exists");
579 return false;
580 }
581 INode dstParent = dstIIP.getINode(-2);
582 if (dstParent == null) {
583 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
584 +"failed to rename "+src+" to "+dst+
585 " because destination's parent does not exist");
586 return false;
587 }
588
589 // Ensure dst has quota to accommodate rename
590 verifyQuotaForRename(srcIIP.getINodes(), dstIIP.getINodes());
591
592 boolean added = false;
593 INode srcChild = srcIIP.getLastINode();
594 final byte[] srcChildName = srcChild.getLocalNameBytes();
595 final boolean isSrcInSnapshot = srcChild.isInLatestSnapshot(
596 srcIIP.getLatestSnapshot());
597 final boolean srcChildIsReference = srcChild.isReference();
598
599 // Record the snapshot on srcChild. After the rename, before any new
600 // snapshot is taken on the dst tree, changes will be recorded in the latest
601 // snapshot of the src tree.
602 if (isSrcInSnapshot) {
603 srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot(),
604 inodeMap);
605 srcIIP.setLastINode(srcChild);
606 }
607
608 // check srcChild for reference
609 final INodeReference.WithCount withCount;
610 Quota.Counts oldSrcCounts = Quota.Counts.newInstance();
611 int srcRefDstSnapshot = srcChildIsReference ? srcChild.asReference()
612 .getDstSnapshotId() : Snapshot.INVALID_ID;
613 if (isSrcInSnapshot) {
614 final INodeReference.WithName withName =
615 srcIIP.getINode(-2).asDirectory().replaceChild4ReferenceWithName(
616 srcChild, srcIIP.getLatestSnapshot());
617 withCount = (INodeReference.WithCount) withName.getReferredINode();
618 srcChild = withName;
619 srcIIP.setLastINode(srcChild);
620 // get the counts before rename
621 withCount.getReferredINode().computeQuotaUsage(oldSrcCounts, true,
622 Snapshot.INVALID_ID);
623 } else if (srcChildIsReference) {
624 // srcChild is reference but srcChild is not in latest snapshot
625 withCount = (WithCount) srcChild.asReference().getReferredINode();
626 } else {
627 withCount = null;
628 }
629
630 try {
631 // remove src
632 final long removedSrc = removeLastINode(srcIIP);
633 if (removedSrc == -1) {
634 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
635 + "failed to rename " + src + " to " + dst
636 + " because the source can not be removed");
637 return false;
638 }
639
640 if (dstParent.getParent() == null) {
641 // src and dst file/dir are in the same directory, and the dstParent has
642 // been replaced when we removed the src. Refresh the dstIIP and
643 // dstParent.
644 dstIIP = getExistingPathINodes(dstComponents);
645 dstParent = dstIIP.getINode(-2);
646 }
647
648 // add src to the destination
649
650 srcChild = srcIIP.getLastINode();
651 final byte[] dstChildName = dstIIP.getLastLocalName();
652 final INode toDst;
653 if (withCount == null) {
654 srcChild.setLocalName(dstChildName);
655 toDst = srcChild;
656 } else {
657 withCount.getReferredINode().setLocalName(dstChildName);
658 Snapshot dstSnapshot = dstIIP.getLatestSnapshot();
659 final INodeReference.DstReference ref = new INodeReference.DstReference(
660 dstParent.asDirectory(), withCount,
661 dstSnapshot == null ? Snapshot.INVALID_ID : dstSnapshot.getId());
662 toDst = ref;
663 }
664
665 added = addLastINodeNoQuotaCheck(dstIIP, toDst);
666 if (added) {
667 if (NameNode.stateChangeLog.isDebugEnabled()) {
668 NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedRenameTo: "
669 + src + " is renamed to " + dst);
670 }
671 // update modification time of dst and the parent of src
672 final INode srcParent = srcIIP.getINode(-2);
673 srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot(),
674 inodeMap);
675 dstParent = dstIIP.getINode(-2); // refresh dstParent
676 dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot(),
677 inodeMap);
678 // update moved leases with new filename
679 getFSNamesystem().unprotectedChangeLease(src, dst);
680
681 // update the quota usage in src tree
682 if (isSrcInSnapshot) {
683 // get the counts after rename
684 Quota.Counts newSrcCounts = srcChild.computeQuotaUsage(
685 Quota.Counts.newInstance(), false, Snapshot.INVALID_ID);
686 newSrcCounts.subtract(oldSrcCounts);
687 srcParent.addSpaceConsumed(newSrcCounts.get(Quota.NAMESPACE),
688 newSrcCounts.get(Quota.DISKSPACE), false);
689 }
690
691 return true;
692 }
693 } finally {
694 if (!added) {
695 final INodeDirectory srcParent = srcIIP.getINode(-2).asDirectory();
696 final INode oldSrcChild = srcChild;
697 // put it back
698 if (withCount == null) {
699 srcChild.setLocalName(srcChildName);
700 } else if (!srcChildIsReference) { // src must be in snapshot
701 // the withCount node will no longer be used thus no need to update
702 // its reference number here
703 final INode originalChild = withCount.getReferredINode();
704 srcChild = originalChild;
705 srcChild.setLocalName(srcChildName);
706 } else {
707 withCount.removeReference(oldSrcChild.asReference());
708 final INodeReference originalRef = new INodeReference.DstReference(
709 srcParent, withCount, srcRefDstSnapshot);
710 srcChild = originalRef;
711 withCount.getReferredINode().setLocalName(srcChildName);
712 }
713
714 if (isSrcInSnapshot) {
715 // srcParent must be an INodeDirectoryWithSnapshot instance since
716 // isSrcInSnapshot is true and src node has been removed from
717 // srcParent
718 ((INodeDirectoryWithSnapshot) srcParent).undoRename4ScrParent(
719 oldSrcChild.asReference(), srcChild, srcIIP.getLatestSnapshot());
720 } else {
721 // original srcChild is not in latest snapshot, we only need to add
722 // the srcChild back
723 addLastINodeNoQuotaCheck(srcIIP, srcChild);
724 }
725 }
726 }
727 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
728 +"failed to rename "+src+" to "+dst);
729 return false;
730 }
731
732 /**
733 * Rename src to dst.
734 * See {@link DistributedFileSystem#rename(Path, Path, Options.Rename...)}
735 * for details related to rename semantics and exceptions.
736 *
737 * @param src source path
738 * @param dst destination path
739 * @param timestamp modification time
740 * @param options Rename options
741 */
742 boolean unprotectedRenameTo(String src, String dst, long timestamp,
743 Options.Rename... options) throws FileAlreadyExistsException,
744 FileNotFoundException, ParentNotDirectoryException,
745 QuotaExceededException, UnresolvedLinkException, IOException {
746 assert hasWriteLock();
747 boolean overwrite = false;
748 if (null != options) {
749 for (Rename option : options) {
750 if (option == Rename.OVERWRITE) {
751 overwrite = true;
752 }
753 }
754 }
755 String error = null;
756 final INodesInPath srcIIP = rootDir.getINodesInPath4Write(src, false);
757 final INode srcInode = srcIIP.getLastINode();
758 // validate source
759 if (srcInode == null) {
760 error = "rename source " + src + " is not found.";
761 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
762 + error);
763 throw new FileNotFoundException(error);
764 }
765 if (srcIIP.getINodes().length == 1) {
766 error = "rename source cannot be the root";
767 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
768 + error);
769 throw new IOException(error);
770 }
771 // srcInode and its subtree cannot contain snapshottable directories with
772 // snapshots
773 checkSnapshot(srcInode, null);
774
775 // validate the destination
776 if (dst.equals(src)) {
777 throw new FileAlreadyExistsException(
778 "The source "+src+" and destination "+dst+" are the same");
779 }
780 if (srcInode.isSymlink() &&
781 dst.equals(srcInode.asSymlink().getSymlinkString())) {
782 throw new FileAlreadyExistsException(
783 "Cannot rename symlink "+src+" to its target "+dst);
784 }
785 // dst cannot be a directory or a file under src
786 if (dst.startsWith(src) &&
787 dst.charAt(src.length()) == Path.SEPARATOR_CHAR) {
788 error = "Rename destination " + dst
789 + " is a directory or file under source " + src;
790 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
791 + error);
792 throw new IOException(error);
793 }
794 INodesInPath dstIIP = rootDir.getINodesInPath4Write(dst, false);
795 if (dstIIP.getINodes().length == 1) {
796 error = "rename destination cannot be the root";
797 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
798 + error);
799 throw new IOException(error);
800 }
801
802 final INode dstInode = dstIIP.getLastINode();
803 List<INodeDirectorySnapshottable> snapshottableDirs =
804 new ArrayList<INodeDirectorySnapshottable>();
805 if (dstInode != null) { // Destination exists
806 // It's OK to rename a file to a symlink and vice versa
807 if (dstInode.isDirectory() != srcInode.isDirectory()) {
808 error = "Source " + src + " and destination " + dst
809 + " must both be directories";
810 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
811 + error);
812 throw new IOException(error);
813 }
814 if (!overwrite) { // If destination exists, overwrite flag must be true
815 error = "rename destination " + dst + " already exists";
816 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
817 + error);
818 throw new FileAlreadyExistsException(error);
819 }
820 if (dstInode.isDirectory()) {
821 final ReadOnlyList<INode> children = dstInode.asDirectory()
822 .getChildrenList(null);
823 if (!children.isEmpty()) {
824 error = "rename destination directory is not empty: " + dst;
825 NameNode.stateChangeLog.warn(
826 "DIR* FSDirectory.unprotectedRenameTo: " + error);
827 throw new IOException(error);
828 }
829 }
830 checkSnapshot(dstInode, snapshottableDirs);
831 }
832
833 INode dstParent = dstIIP.getINode(-2);
834 if (dstParent == null) {
835 error = "rename destination parent " + dst + " not found.";
836 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
837 + error);
838 throw new FileNotFoundException(error);
839 }
840 if (!dstParent.isDirectory()) {
841 error = "rename destination parent " + dst + " is a file.";
842 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
843 + error);
844 throw new ParentNotDirectoryException(error);
845 }
846
847 // Ensure dst has quota to accommodate rename
848 verifyQuotaForRename(srcIIP.getINodes(), dstIIP.getINodes());
849
850 INode srcChild = srcIIP.getLastINode();
851 final byte[] srcChildName = srcChild.getLocalNameBytes();
852 final boolean isSrcInSnapshot = srcChild.isInLatestSnapshot(
853 srcIIP.getLatestSnapshot());
854 final boolean srcChildIsReference = srcChild.isReference();
855
856 // Record the snapshot on srcChild. After the rename, before any new
857 // snapshot is taken on the dst tree, changes will be recorded in the latest
858 // snapshot of the src tree.
859 if (isSrcInSnapshot) {
860 srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot(),
861 inodeMap);
862 srcIIP.setLastINode(srcChild);
863 }
864
865 // check srcChild for reference
866 final INodeReference.WithCount withCount;
867 int srcRefDstSnapshot = srcChildIsReference ? srcChild.asReference()
868 .getDstSnapshotId() : Snapshot.INVALID_ID;
869 Quota.Counts oldSrcCounts = Quota.Counts.newInstance();
870 if (isSrcInSnapshot) {
871 final INodeReference.WithName withName = srcIIP.getINode(-2).asDirectory()
872 .replaceChild4ReferenceWithName(srcChild, srcIIP.getLatestSnapshot());
873 withCount = (INodeReference.WithCount) withName.getReferredINode();
874 srcChild = withName;
875 srcIIP.setLastINode(srcChild);
876 // get the counts before rename
877 withCount.getReferredINode().computeQuotaUsage(oldSrcCounts, true,
878 Snapshot.INVALID_ID);
879 } else if (srcChildIsReference) {
880 // srcChild is reference but srcChild is not in latest snapshot
881 withCount = (WithCount) srcChild.asReference().getReferredINode();
882 } else {
883 withCount = null;
884 }
885
886 boolean undoRemoveSrc = true;
887 final long removedSrc = removeLastINode(srcIIP);
888 if (removedSrc == -1) {
889 error = "Failed to rename " + src + " to " + dst
890 + " because the source can not be removed";
891 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
892 + error);
893 throw new IOException(error);
894 }
895
896 if (dstParent.getParent() == null) {
897 // src and dst file/dir are in the same directory, and the dstParent has
898 // been replaced when we removed the src. Refresh the dstIIP and
899 // dstParent.
900 dstIIP = rootDir.getINodesInPath4Write(dst, false);
901 }
902
903 boolean undoRemoveDst = false;
904 INode removedDst = null;
905 try {
906 if (dstInode != null) { // dst exists remove it
907 if (removeLastINode(dstIIP) != -1) {
908 removedDst = dstIIP.getLastINode();
909 undoRemoveDst = true;
910 }
911 }
912
913 srcChild = srcIIP.getLastINode();
914
915 final byte[] dstChildName = dstIIP.getLastLocalName();
916 final INode toDst;
917 if (withCount == null) {
918 srcChild.setLocalName(dstChildName);
919 toDst = srcChild;
920 } else {
921 withCount.getReferredINode().setLocalName(dstChildName);
922 Snapshot dstSnapshot = dstIIP.getLatestSnapshot();
923 final INodeReference.DstReference ref = new INodeReference.DstReference(
924 dstIIP.getINode(-2).asDirectory(), withCount,
925 dstSnapshot == null ? Snapshot.INVALID_ID : dstSnapshot.getId());
926 toDst = ref;
927 }
928
929 // add src as dst to complete rename
930 if (addLastINodeNoQuotaCheck(dstIIP, toDst)) {
931 undoRemoveSrc = false;
932 if (NameNode.stateChangeLog.isDebugEnabled()) {
933 NameNode.stateChangeLog.debug(
934 "DIR* FSDirectory.unprotectedRenameTo: " + src
935 + " is renamed to " + dst);
936 }
937
938 final INode srcParent = srcIIP.getINode(-2);
939 srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot(),
940 inodeMap);
941 dstParent = dstIIP.getINode(-2);
942 dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot(),
943 inodeMap);
944 // update moved lease with new filename
945 getFSNamesystem().unprotectedChangeLease(src, dst);
946
947 // Collect the blocks and remove the lease for previous dst
948 long filesDeleted = -1;
949 if (removedDst != null) {
950 undoRemoveDst = false;
951 BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
952 List<INode> removedINodes = new ArrayList<INode>();
953 filesDeleted = removedDst.cleanSubtree(null,
954 dstIIP.getLatestSnapshot(), collectedBlocks, removedINodes, true)
955 .get(Quota.NAMESPACE);
956 getFSNamesystem().removePathAndBlocks(src, collectedBlocks,
957 removedINodes);
958 }
959
960 if (snapshottableDirs.size() > 0) {
961 // There are snapshottable directories (without snapshots) to be
962 // deleted. Need to update the SnapshotManager.
963 namesystem.removeSnapshottableDirs(snapshottableDirs);
964 }
965
966 // update the quota usage in src tree
967 if (isSrcInSnapshot) {
968 // get the counts after rename
969 Quota.Counts newSrcCounts = srcChild.computeQuotaUsage(
970 Quota.Counts.newInstance(), false, Snapshot.INVALID_ID);
971 newSrcCounts.subtract(oldSrcCounts);
972 srcParent.addSpaceConsumed(newSrcCounts.get(Quota.NAMESPACE),
973 newSrcCounts.get(Quota.DISKSPACE), false);
974 }
975
976 return filesDeleted >= 0;
977 }
978 } finally {
979 if (undoRemoveSrc) {
980 // Rename failed - restore src
981 final INodeDirectory srcParent = srcIIP.getINode(-2).asDirectory();
982 final INode oldSrcChild = srcChild;
983 // put it back
984 if (withCount == null) {
985 srcChild.setLocalName(srcChildName);
986 } else if (!srcChildIsReference) { // src must be in snapshot
987 // the withCount node will no longer be used thus no need to update
988 // its reference number here
989 final INode originalChild = withCount.getReferredINode();
990 srcChild = originalChild;
991 srcChild.setLocalName(srcChildName);
992 } else {
993 withCount.removeReference(oldSrcChild.asReference());
994 final INodeReference originalRef = new INodeReference.DstReference(
995 srcParent, withCount, srcRefDstSnapshot);
996 srcChild = originalRef;
997 withCount.getReferredINode().setLocalName(srcChildName);
998 }
999
1000 if (srcParent instanceof INodeDirectoryWithSnapshot) {
1001 ((INodeDirectoryWithSnapshot) srcParent).undoRename4ScrParent(
1002 oldSrcChild.asReference(), srcChild, srcIIP.getLatestSnapshot());
1003 } else {
1004 // srcParent is not an INodeDirectoryWithSnapshot, we only need to add
1005 // the srcChild back
1006 addLastINodeNoQuotaCheck(srcIIP, srcChild);
1007 }
1008 }
1009 if (undoRemoveDst) {
1010 // Rename failed - restore dst
1011 if (dstParent instanceof INodeDirectoryWithSnapshot) {
1012 ((INodeDirectoryWithSnapshot) dstParent).undoRename4DstParent(
1013 removedDst, dstIIP.getLatestSnapshot());
1014 } else {
1015 addLastINodeNoQuotaCheck(dstIIP, removedDst);
1016 }
1017 if (removedDst.isReference()) {
1018 final INodeReference removedDstRef = removedDst.asReference();
1019 final INodeReference.WithCount wc =
1020 (WithCount) removedDstRef.getReferredINode().asReference();
1021 wc.addReference(removedDstRef);
1022 }
1023 }
1024 }
1025 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
1026 + "failed to rename " + src + " to " + dst);
1027 throw new IOException("rename from " + src + " to " + dst + " failed.");
1028 }
1029
1030 /**
1031 * Set file replication
1032 *
1033 * @param src file name
1034 * @param replication new replication
1035 * @param blockRepls block replications - output parameter
1036 * @return array of file blocks
1037 * @throws QuotaExceededException
1038 * @throws SnapshotAccessControlException
1039 */
1040 Block[] setReplication(String src, short replication, short[] blockRepls)
1041 throws QuotaExceededException, UnresolvedLinkException,
1042 SnapshotAccessControlException {
1043 waitForReady();
1044 writeLock();
1045 try {
1046 final Block[] fileBlocks = unprotectedSetReplication(
1047 src, replication, blockRepls);
1048 if (fileBlocks != null) // log replication change
1049 fsImage.getEditLog().logSetReplication(src, replication);
1050 return fileBlocks;
1051 } finally {
1052 writeUnlock();
1053 }
1054 }
1055
1056 Block[] unprotectedSetReplication(String src, short replication,
1057 short[] blockRepls) throws QuotaExceededException,
1058 UnresolvedLinkException, SnapshotAccessControlException {
1059 assert hasWriteLock();
1060
1061 final INodesInPath iip = rootDir.getINodesInPath4Write(src, true);
1062 final INode inode = iip.getLastINode();
1063 if (inode == null || !inode.isFile()) {
1064 return null;
1065 }
1066 INodeFile file = inode.asFile();
1067 final short oldBR = file.getBlockReplication();
1068
1069 // before setFileReplication, check for increasing block replication.
1070 // if replication > oldBR, then newBR == replication.
1071 // if replication < oldBR, we don't know newBR yet.
1072 if (replication > oldBR) {
1073 long dsDelta = (replication - oldBR)*(file.diskspaceConsumed()/oldBR);
1074 updateCount(iip, 0, dsDelta, true);
1075 }
1076
1077 file = file.setFileReplication(replication, iip.getLatestSnapshot(),
1078 inodeMap);
1079
1080 final short newBR = file.getBlockReplication();
1081 // check newBR < oldBR case.
1082 if (newBR < oldBR) {
1083 long dsDelta = (newBR - oldBR)*(file.diskspaceConsumed()/newBR);
1084 updateCount(iip, 0, dsDelta, true);
1085 }
1086
1087 if (blockRepls != null) {
1088 blockRepls[0] = oldBR;
1089 blockRepls[1] = newBR;
1090 }
1091 return file.getBlocks();
1092 }
1093
1094 /**
1095 * @param path the file path
1096 * @return the block size of the file.
1097 */
1098 long getPreferredBlockSize(String path) throws UnresolvedLinkException,
1099 FileNotFoundException, IOException {
1100 readLock();
1101 try {
1102 return INodeFile.valueOf(rootDir.getNode(path, false), path
1103 ).getPreferredBlockSize();
1104 } finally {
1105 readUnlock();
1106 }
1107 }
1108
1109 boolean exists(String src) throws UnresolvedLinkException {
1110 src = normalizePath(src);
1111 readLock();
1112 try {
1113 INode inode = rootDir.getNode(src, false);
1114 if (inode == null) {
1115 return false;
1116 }
1117 return !inode.isFile() || inode.asFile().getBlocks() != null;
1118 } finally {
1119 readUnlock();
1120 }
1121 }
1122
1123 void setPermission(String src, FsPermission permission)
1124 throws FileNotFoundException, UnresolvedLinkException,
1125 QuotaExceededException, SnapshotAccessControlException {
1126 writeLock();
1127 try {
1128 unprotectedSetPermission(src, permission);
1129 } finally {
1130 writeUnlock();
1131 }
1132 fsImage.getEditLog().logSetPermissions(src, permission);
1133 }
1134
1135 void unprotectedSetPermission(String src, FsPermission permissions)
1136 throws FileNotFoundException, UnresolvedLinkException,
1137 QuotaExceededException, SnapshotAccessControlException {
1138 assert hasWriteLock();
1139 final INodesInPath inodesInPath = rootDir.getINodesInPath4Write(src, true);
1140 final INode inode = inodesInPath.getLastINode();
1141 if (inode == null) {
1142 throw new FileNotFoundException("File does not exist: " + src);
1143 }
1144 inode.setPermission(permissions, inodesInPath.getLatestSnapshot(),
1145 inodeMap);
1146 }
1147
1148 void setOwner(String src, String username, String groupname)
1149 throws FileNotFoundException, UnresolvedLinkException,
1150 QuotaExceededException, SnapshotAccessControlException {
1151 writeLock();
1152 try {
1153 unprotectedSetOwner(src, username, groupname);
1154 } finally {
1155 writeUnlock();
1156 }
1157 fsImage.getEditLog().logSetOwner(src, username, groupname);
1158 }
1159
1160 void unprotectedSetOwner(String src, String username, String groupname)
1161 throws FileNotFoundException, UnresolvedLinkException,
1162 QuotaExceededException, SnapshotAccessControlException {
1163 assert hasWriteLock();
1164 final INodesInPath inodesInPath = rootDir.getINodesInPath4Write(src, true);
1165 INode inode = inodesInPath.getLastINode();
1166 if (inode == null) {
1167 throw new FileNotFoundException("File does not exist: " + src);
1168 }
1169 if (username != null) {
1170 inode = inode.setUser(username, inodesInPath.getLatestSnapshot(),
1171 inodeMap);
1172 }
1173 if (groupname != null) {
1174 inode.setGroup(groupname, inodesInPath.getLatestSnapshot(), inodeMap);
1175 }
1176 }
1177
1178 /**
1179 * Concat all the blocks from srcs to trg and delete the srcs files
1180 */
1181 void concat(String target, String [] srcs, boolean supportRetryCache)
1182 throws UnresolvedLinkException, QuotaExceededException,
1183 SnapshotAccessControlException, SnapshotException {
1184 writeLock();
1185 try {
1186 // actual move
1187 waitForReady();
1188 long timestamp = now();
1189 unprotectedConcat(target, srcs, timestamp);
1190 // do the commit
1191 fsImage.getEditLog().logConcat(target, srcs, timestamp,
1192 supportRetryCache);
1193 } finally {
1194 writeUnlock();
1195 }
1196 }
1197
1198 /**
1199 * Concat all the blocks from srcs to trg and delete the srcs files
1200 * @param target target file to move the blocks to
1201 * @param srcs list of file to move the blocks from
1202 */
1203 void unprotectedConcat(String target, String [] srcs, long timestamp)
1204 throws UnresolvedLinkException, QuotaExceededException,
1205 SnapshotAccessControlException, SnapshotException {
1206 assert hasWriteLock();
1207 if (NameNode.stateChangeLog.isDebugEnabled()) {
1208 NameNode.stateChangeLog.debug("DIR* FSNamesystem.concat to "+target);
1209 }
1210 // do the move
1211
1212 final INodesInPath trgIIP = rootDir.getINodesInPath4Write(target, true);
1213 final INode[] trgINodes = trgIIP.getINodes();
1214 final INodeFile trgInode = trgIIP.getLastINode().asFile();
1215 INodeDirectory trgParent = trgINodes[trgINodes.length-2].asDirectory();
1216 final Snapshot trgLatestSnapshot = trgIIP.getLatestSnapshot();
1217
1218 final INodeFile [] allSrcInodes = new INodeFile[srcs.length];
1219 for(int i = 0; i < srcs.length; i++) {
1220 final INodesInPath iip = getINodesInPath4Write(srcs[i]);
1221 final Snapshot latest = iip.getLatestSnapshot();
1222 final INode inode = iip.getLastINode();
1223
1224 // check if the file in the latest snapshot
1225 if (inode.isInLatestSnapshot(latest)) {
1226 throw new SnapshotException("Concat: the source file " + srcs[i]
1227 + " is in snapshot " + latest);
1228 }
1229
1230 // check if the file has other references.
1231 if (inode.isReference() && ((INodeReference.WithCount)
1232 inode.asReference().getReferredINode()).getReferenceCount() > 1) {
1233 throw new SnapshotException("Concat: the source file " + srcs[i]
1234 + " is referred by some other reference in some snapshot.");
1235 }
1236
1237 allSrcInodes[i] = inode.asFile();
1238 }
1239 trgInode.concatBlocks(allSrcInodes);
1240
1241 // since we are in the same dir - we can use same parent to remove files
1242 int count = 0;
1243 for(INodeFile nodeToRemove: allSrcInodes) {
1244 if(nodeToRemove == null) continue;
1245
1246 nodeToRemove.setBlocks(null);
1247 trgParent.removeChild(nodeToRemove, trgLatestSnapshot, null);
1248 inodeMap.remove(nodeToRemove);
1249 count++;
1250 }
1251
1252 // update inodeMap
1253 removeFromInodeMap(Arrays.asList(allSrcInodes));
1254
1255 trgInode.setModificationTime(timestamp, trgLatestSnapshot, inodeMap);
1256 trgParent.updateModificationTime(timestamp, trgLatestSnapshot, inodeMap);
1257 // update quota on the parent directory ('count' files removed, 0 space)
1258 unprotectedUpdateCount(trgIIP, trgINodes.length-1, -count, 0);
1259 }
1260
1261 /**
1262 * Delete the target directory and collect the blocks under it
1263 *
1264 * @param src Path of a directory to delete
1265 * @param collectedBlocks Blocks under the deleted directory
1266 * @param removedINodes INodes that should be removed from {@link #inodeMap}
1267 * @param logRetryCache Whether to record RPC IDs in editlog to support retry
1268 * cache rebuilding.
1269 * @return true on successful deletion; else false
1270 */
1271 boolean delete(String src, BlocksMapUpdateInfo collectedBlocks,
1272 List<INode> removedINodes, boolean logRetryCache) throws IOException {
1273 if (NameNode.stateChangeLog.isDebugEnabled()) {
1274 NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
1275 }
1276 waitForReady();
1277 long now = now();
1278 final long filesRemoved;
1279 writeLock();
1280 try {
1281 final INodesInPath inodesInPath = rootDir.getINodesInPath4Write(
1282 normalizePath(src), false);
1283 if (!deleteAllowed(inodesInPath, src) ) {
1284 filesRemoved = -1;
1285 } else {
1286 // Before removing the node, first check if the targetNode is for a
1287 // snapshottable dir with snapshots, or its descendants have
1288 // snapshottable dir with snapshots
1289 final INode targetNode = inodesInPath.getLastINode();
1290 List<INodeDirectorySnapshottable> snapshottableDirs =
1291 new ArrayList<INodeDirectorySnapshottable>();
1292 checkSnapshot(targetNode, snapshottableDirs);
1293 filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
1294 removedINodes, now);
1295 if (snapshottableDirs.size() > 0) {
1296 // There are some snapshottable directories without snapshots to be
1297 // deleted. Need to update the SnapshotManager.
1298 namesystem.removeSnapshottableDirs(snapshottableDirs);
1299 }
1300 }
1301 } finally {
1302 writeUnlock();
1303 }
1304 if (filesRemoved < 0) {
1305 return false;
1306 }
1307 fsImage.getEditLog().logDelete(src, now, logRetryCache);
1308 incrDeletedFileCount(filesRemoved);
1309 // Blocks/INodes will be handled later by the caller of this method
1310 getFSNamesystem().removePathAndBlocks(src, null, null);
1311 return true;
1312 }
1313
1314 private static boolean deleteAllowed(final INodesInPath iip,
1315 final String src) {
1316 final INode[] inodes = iip.getINodes();
1317 if (inodes == null || inodes.length == 0
1318 || inodes[inodes.length - 1] == null) {
1319 if(NameNode.stateChangeLog.isDebugEnabled()) {
1320 NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
1321 + "failed to remove " + src + " because it does not exist");
1322 }
1323 return false;
1324 } else if (inodes.length == 1) { // src is the root
1325 NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedDelete: "
1326 + "failed to remove " + src
1327 + " because the root is not allowed to be deleted");
1328 return false;
1329 }
1330 return true;
1331 }
1332
1333 /**
1334 * @return true if the path is a non-empty directory; otherwise, return false.
1335 */
1336 boolean isNonEmptyDirectory(String path) throws UnresolvedLinkException {
1337 readLock();
1338 try {
1339 final INodesInPath inodesInPath = rootDir.getLastINodeInPath(path, false);
1340 final INode inode = inodesInPath.getINode(0);
1341 if (inode == null || !inode.isDirectory()) {
1342 //not found or not a directory
1343 return false;
1344 }
1345 final Snapshot s = inodesInPath.getPathSnapshot();
1346 return !inode.asDirectory().getChildrenList(s).isEmpty();
1347 } finally {
1348 readUnlock();
1349 }
1350 }
1351
1352 /**
1353 * Delete a path from the name space
1354 * Update the count at each ancestor directory with quota
1355 * <br>
1356 * Note: This is to be used by {@link FSEditLog} only.
1357 * <br>
1358 * @param src a string representation of a path to an inode
1359 * @param mtime the time the inode is removed
1360 * @throws SnapshotAccessControlException if path is in RO snapshot
1361 */
1362 void unprotectedDelete(String src, long mtime) throws UnresolvedLinkException,
1363 QuotaExceededException, SnapshotAccessControlException {
1364 assert hasWriteLock();
1365 BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
1366 List<INode> removedINodes = new ArrayList<INode>();
1367
1368 final INodesInPath inodesInPath = rootDir.getINodesInPath4Write(
1369 normalizePath(src), false);
1370 final long filesRemoved = deleteAllowed(inodesInPath, src) ?
1371 unprotectedDelete(inodesInPath, collectedBlocks,
1372 removedINodes, mtime) : -1;
1373 if (filesRemoved >= 0) {
1374 getFSNamesystem().removePathAndBlocks(src, collectedBlocks,
1375 removedINodes);
1376 }
1377 }
1378
1379 /**
1380 * Delete a path from the name space
1381 * Update the count at each ancestor directory with quota
1382 * @param iip the inodes resolved from the path
1383 * @param collectedBlocks blocks collected from the deleted path
1384 * @param removedINodes inodes that should be removed from {@link #inodeMap}
1385 * @param mtime the time the inode is removed
1386 * @return the number of inodes deleted; 0 if no inodes are deleted.
1387 */
1388 long unprotectedDelete(INodesInPath iip, BlocksMapUpdateInfo collectedBlocks,
1389 List<INode> removedINodes, long mtime) throws QuotaExceededException {
1390 assert hasWriteLock();
1391
1392 // check if target node exists
1393 INode targetNode = iip.getLastINode();
1394 if (targetNode == null) {
1395 return -1;
1396 }
1397
1398 // record modification
1399 final Snapshot latestSnapshot = iip.getLatestSnapshot();
1400 targetNode = targetNode.recordModification(latestSnapshot, inodeMap);
1401 iip.setLastINode(targetNode);
1402
1403 // Remove the node from the namespace
1404 long removed = removeLastINode(iip);
1405 if (removed == -1) {
1406 return -1;
1407 }
1408
1409 // set the parent's modification time
1410 final INodeDirectory parent = targetNode.getParent();
1411 parent.updateModificationTime(mtime, latestSnapshot, inodeMap);
1412 if (removed == 0) {
1413 return 0;
1414 }
1415
1416 // collect block
1417 if (!targetNode.isInLatestSnapshot(latestSnapshot)) {
1418 targetNode.destroyAndCollectBlocks(collectedBlocks, removedINodes);
1419 } else {
1420 Quota.Counts counts = targetNode.cleanSubtree(null, latestSnapshot,
1421 collectedBlocks, removedINodes, true);
1422 parent.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
1423 -counts.get(Quota.DISKSPACE), true);
1424 removed = counts.get(Quota.NAMESPACE);
1425 }
1426 if (NameNode.stateChangeLog.isDebugEnabled()) {
1427 NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
1428 + targetNode.getFullPathName() + " is removed");
1429 }
1430 return removed;
1431 }
1432
1433 /**
1434 * Check if the given INode (or one of its descendants) is snapshottable and
1435 * already has snapshots.
1436 *
1437 * @param target The given INode
1438 * @param snapshottableDirs The list of directories that are snapshottable
1439 * but do not have snapshots yet
1440 */
1441 private static void checkSnapshot(INode target,
1442 List<INodeDirectorySnapshottable> snapshottableDirs) throws IOException {
1443 if (target.isDirectory()) {
1444 INodeDirectory targetDir = target.asDirectory();
1445 if (targetDir.isSnapshottable()) {
1446 INodeDirectorySnapshottable ssTargetDir =
1447 (INodeDirectorySnapshottable) targetDir;
1448 if (ssTargetDir.getNumSnapshots() > 0) {
1449 throw new IOException("The directory " + ssTargetDir.getFullPathName()
1450 + " cannot be deleted since " + ssTargetDir.getFullPathName()
1451 + " is snapshottable and already has snapshots");
1452 } else {
1453 if (snapshottableDirs != null) {
1454 snapshottableDirs.add(ssTargetDir);
1455 }
1456 }
1457 }
1458 for (INode child : targetDir.getChildrenList(null)) {
1459 checkSnapshot(child, snapshottableDirs);
1460 }
1461 }
1462 }
1463
1464 /**
1465 * Replaces the specified INodeFile with the specified one.
1466 */
1467 void replaceINodeFile(String path, INodeFile oldnode,
1468 INodeFile newnode) throws IOException {
1469 writeLock();
1470 try {
1471 unprotectedReplaceINodeFile(path, oldnode, newnode);
1472 } finally {
1473 writeUnlock();
1474 }
1475 }
1476
1477 /** Replace an INodeFile and record modification for the latest snapshot. */
1478 void unprotectedReplaceINodeFile(final String path, final INodeFile oldnode,
1479 final INodeFile newnode) {
1480 Preconditions.checkState(hasWriteLock());
1481
1482 oldnode.getParent().replaceChild(oldnode, newnode, inodeMap);
1483 oldnode.clear();
1484
1485 /* Currently oldnode and newnode are assumed to contain the same
1486 * blocks. Otherwise, blocks need to be removed from the blocksMap.
1487 */
1488 int index = 0;
1489 for (BlockInfo b : newnode.getBlocks()) {
1490 BlockInfo info = getBlockManager().addBlockCollection(b, newnode);
1491 newnode.setBlock(index, info); // inode refers to the block in BlocksMap
1492 index++;
1493 }
1494 }
1495
1496 /**
1497 * Get a partial listing of the indicated directory
1498 *
1499 * @param src the directory name
1500 * @param startAfter the name to start listing after
1501 * @param needLocation if block locations are returned
1502 * @return a partial listing starting after startAfter
1503 */
1504 DirectoryListing getListing(String src, byte[] startAfter,
1505 boolean needLocation) throws UnresolvedLinkException, IOException {
1506 String srcs = normalizePath(src);
1507
1508 readLock();
1509 try {
1510 if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
1511 return getSnapshotsListing(srcs, startAfter);
1512 }
1513 final INodesInPath inodesInPath = rootDir.getLastINodeInPath(srcs, true);
1514 final Snapshot snapshot = inodesInPath.getPathSnapshot();
1515 final INode targetNode = inodesInPath.getINode(0);
1516 if (targetNode == null)
1517 return null;
1518
1519 if (!targetNode.isDirectory()) {
1520 return new DirectoryListing(
1521 new HdfsFileStatus[]{createFileStatus(HdfsFileStatus.EMPTY_NAME,
1522 targetNode, needLocation, snapshot)}, 0);
1523 }
1524
1525 final INodeDirectory dirInode = targetNode.asDirectory();
1526 final ReadOnlyList<INode> contents = dirInode.getChildrenList(snapshot);
1527 int startChild = INodeDirectory.nextChild(contents, startAfter);
1528 int totalNumChildren = contents.size();
1529 int numOfListing = Math.min(totalNumChildren-startChild, this.lsLimit);
1530 HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
1531 for (int i=0; i<numOfListing; i++) {
1532 INode cur = contents.get(startChild+i);
1533 listing[i] = createFileStatus(cur.getLocalNameBytes(), cur,
1534 needLocation, snapshot);
1535 }
1536 return new DirectoryListing(
1537 listing, totalNumChildren-startChild-numOfListing);
1538 } finally {
1539 readUnlock();
1540 }
1541 }
1542
1543 /**
1544 * Get a listing of all the snapshots of a snapshottable directory
1545 */
1546 private DirectoryListing getSnapshotsListing(String src, byte[] startAfter)
1547 throws UnresolvedLinkException, IOException {
1548 Preconditions.checkState(hasReadLock());
1549 Preconditions.checkArgument(
1550 src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
1551 "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
1552
1553 final String dirPath = normalizePath(src.substring(0,
1554 src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
1555
1556 final INode node = this.getINode(dirPath);
1557 final INodeDirectorySnapshottable dirNode = INodeDirectorySnapshottable
1558 .valueOf(node, dirPath);
1559 final ReadOnlyList<Snapshot> snapshots = dirNode.getSnapshotList();
1560 int skipSize = ReadOnlyList.Util.binarySearch(snapshots, startAfter);
1561 skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1;
1562 int numOfListing = Math.min(snapshots.size() - skipSize, this.lsLimit);
1563 final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
1564 for (int i = 0; i < numOfListing; i++) {
1565 Root sRoot = snapshots.get(i + skipSize).getRoot();
1566 listing[i] = createFileStatus(sRoot.getLocalNameBytes(), sRoot, null);
1567 }
1568 return new DirectoryListing(
1569 listing, snapshots.size() - skipSize - numOfListing);
1570 }
1571
1572 /** Get the file info for a specific file.
1573 * @param src The string representation of the path to the file
1574 * @param resolveLink whether to throw UnresolvedLinkException
1575 * @return object containing information regarding the file
1576 * or null if file not found
1577 */
1578 HdfsFileStatus getFileInfo(String src, boolean resolveLink)
1579 throws UnresolvedLinkException {
1580 String srcs = normalizePath(src);
1581 readLock();
1582 try {
1583 if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
1584 return getFileInfo4DotSnapshot(srcs);
1585 }
1586 final INodesInPath inodesInPath = rootDir.getLastINodeInPath(srcs, resolveLink);
1587 final INode i = inodesInPath.getINode(0);
1588 return i == null? null: createFileStatus(HdfsFileStatus.EMPTY_NAME, i,
1589 inodesInPath.getPathSnapshot());
1590 } finally {
1591 readUnlock();
1592 }
1593 }
1594
1595 /**
1596 * Currently we only support "ls /xxx/.snapshot" which will return all the
1597 * snapshots of a directory. The FSCommand Ls will first call getFileInfo to
1598 * make sure the file/directory exists (before the real getListing call).
1599 * Since we do not have a real INode for ".snapshot", we return an empty
1600 * non-null HdfsFileStatus here.
1601 */
1602 private HdfsFileStatus getFileInfo4DotSnapshot(String src)
1603 throws UnresolvedLinkException {
1604 Preconditions.checkArgument(
1605 src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
1606 "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
1607
1608 final String dirPath = normalizePath(src.substring(0,
1609 src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
1610
1611 final INode node = this.getINode(dirPath);
1612 if (node != null
1613 && node.isDirectory()
1614 && node.asDirectory() instanceof INodeDirectorySnapshottable) {
1615 return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
1616 HdfsFileStatus.EMPTY_NAME, -1L, 0);
1617 }
1618 return null;
1619 }
1620
1621 /**
1622 * Get the blocks associated with the file.
1623 */
1624 Block[] getFileBlocks(String src) throws UnresolvedLinkException {
1625 waitForReady();
1626 readLock();
1627 try {
1628 final INode i = rootDir.getNode(src, false);
1629 return i != null && i.isFile()? i.asFile().getBlocks(): null;
1630 } finally {
1631 readUnlock();
1632 }
1633 }
1634
1635
1636 INodesInPath getExistingPathINodes(byte[][] components)
1637 throws UnresolvedLinkException {
1638 return INodesInPath.resolve(rootDir, components);
1639 }
1640
1641 /**
1642 * Get {@link INode} associated with the file / directory.
1643 */
1644 public INode getINode(String src) throws UnresolvedLinkException {
1645 return getLastINodeInPath(src).getINode(0);
1646 }
1647
1648 /**
1649 * Get {@link INode} associated with the file / directory.
1650 */
1651 public INodesInPath getLastINodeInPath(String src)
1652 throws UnresolvedLinkException {
1653 readLock();
1654 try {
1655 return rootDir.getLastINodeInPath(src, true);
1656 } finally {
1657 readUnlock();
1658 }
1659 }
1660
1661 /**
1662 * Get {@link INode} associated with the file / directory.
1663 */
1664 public INodesInPath getINodesInPath4Write(String src
1665 ) throws UnresolvedLinkException, SnapshotAccessControlException {
1666 readLock();
1667 try {
1668 return rootDir.getINodesInPath4Write(src, true);
1669 } finally {
1670 readUnlock();
1671 }
1672 }
1673
1674 /**
1675 * Get {@link INode} associated with the file / directory.
1676 * @throws SnapshotAccessControlException if path is in RO snapshot
1677 */
1678 public INode getINode4Write(String src) throws UnresolvedLinkException,
1679 SnapshotAccessControlException {
1680 readLock();
1681 try {
1682 return rootDir.getINode4Write(src, true);
1683 } finally {
1684 readUnlock();
1685 }
1686 }
1687
1688 /**
1689 * Check whether the filepath could be created
1690 * @throws SnapshotAccessControlException if path is in RO snapshot
1691 */
1692 boolean isValidToCreate(String src) throws UnresolvedLinkException,
1693 SnapshotAccessControlException {
1694 String srcs = normalizePath(src);
1695 readLock();
1696 try {
1697 if (srcs.startsWith("/") && !srcs.endsWith("/")
1698 && rootDir.getINode4Write(srcs, false) == null) {
1699 return true;
1700 } else {
1701 return false;
1702 }
1703 } finally {
1704 readUnlock();
1705 }
1706 }
1707
1708 /**
1709 * Check whether the path specifies a directory
1710 */
1711 boolean isDir(String src) throws UnresolvedLinkException {
1712 src = normalizePath(src);
1713 readLock();
1714 try {
1715 INode node = rootDir.getNode(src, false);
1716 return node != null && node.isDirectory();
1717 } finally {
1718 readUnlock();
1719 }
1720 }
1721
1722 /**
1723 * Check whether the path specifies a directory
1724 * @throws SnapshotAccessControlException if path is in RO snapshot
1725 */
1726 boolean isDirMutable(String src) throws UnresolvedLinkException,
1727 SnapshotAccessControlException {
1728 src = normalizePath(src);
1729 readLock();
1730 try {
1731 INode node = rootDir.getINode4Write(src, false);
1732 return node != null && node.isDirectory();
1733 } finally {
1734 readUnlock();
1735 }
1736 }
1737
1738 /** Updates namespace and diskspace consumed for all
1739 * directories until the parent directory of file represented by path.
1740 *
1741 * @param path path for the file.
1742 * @param nsDelta the delta change of namespace
1743 * @param dsDelta the delta change of diskspace
1744 * @throws QuotaExceededException if the new count violates any quota limit
1745 * @throws FileNotFoundException if path does not exist.
1746 */
1747 void updateSpaceConsumed(String path, long nsDelta, long dsDelta)
1748 throws QuotaExceededException, FileNotFoundException,
1749 UnresolvedLinkException, SnapshotAccessControlException {
1750 writeLock();
1751 try {
1752 final INodesInPath iip = rootDir.getINodesInPath4Write(path, false);
1753 if (iip.getLastINode() == null) {
1754 throw new FileNotFoundException("Path not found: " + path);
1755 }
1756 updateCount(iip, nsDelta, dsDelta, true);
1757 } finally {
1758 writeUnlock();
1759 }
1760 }
1761
1762 private void updateCount(INodesInPath iip, long nsDelta, long dsDelta,
1763 boolean checkQuota) throws QuotaExceededException {
1764 updateCount(iip, iip.getINodes().length - 1, nsDelta, dsDelta, checkQuota);
1765 }
1766
1767 /** update count of each inode with quota
1768 *
1769 * @param iip inodes in a path
1770 * @param numOfINodes the number of inodes to update starting from index 0
1771 * @param nsDelta the delta change of namespace
1772 * @param dsDelta the delta change of diskspace
1773 * @param checkQuota if true then check if quota is exceeded
1774 * @throws QuotaExceededException if the new count violates any quota limit
1775 */
1776 private void updateCount(INodesInPath iip, int numOfINodes,
1777 long nsDelta, long dsDelta, boolean checkQuota)
1778 throws QuotaExceededException {
1779 assert hasWriteLock();
1780 if (!ready) {
1781 //still initializing. do not check or update quotas.
1782 return;
1783 }
1784 final INode[] inodes = iip.getINodes();
1785 if (numOfINodes > inodes.length) {
1786 numOfINodes = inodes.length;
1787 }
1788 if (checkQuota) {
1789 verifyQuota(inodes, numOfINodes, nsDelta, dsDelta, null);
1790 }
1791 unprotectedUpdateCount(iip, numOfINodes, nsDelta, dsDelta);
1792 }
1793
1794 /**
1795 * update quota of each inode and check to see if quota is exceeded.
1796 * See {@link #updateCount(INode[], int, long, long, boolean)}
1797 */
1798 private void updateCountNoQuotaCheck(INodesInPath inodesInPath,
1799 int numOfINodes, long nsDelta, long dsDelta) {
1800 assert hasWriteLock();
1801 try {
1802 updateCount(inodesInPath, numOfINodes, nsDelta, dsDelta, false);
1803 } catch (QuotaExceededException e) {
1804 NameNode.LOG.error("BUG: unexpected exception ", e);
1805 }
1806 }
1807
1808 /**
1809 * updates quota without verification
1810 * callers responsibility is to make sure quota is not exceeded
1811 */
1812 private static void unprotectedUpdateCount(INodesInPath inodesInPath,
1813 int numOfINodes, long nsDelta, long dsDelta) {
1814 final INode[] inodes = inodesInPath.getINodes();
1815 for(int i=0; i < numOfINodes; i++) {
1816 if (inodes[i].isQuotaSet()) { // a directory with quota
1817 INodeDirectoryWithQuota node = (INodeDirectoryWithQuota) inodes[i]
1818 .asDirectory();
1819 node.addSpaceConsumed2Cache(nsDelta, dsDelta);
1820 }
1821 }
1822 }
1823
1824 /** Return the name of the path represented by inodes at [0, pos] */
1825 static String getFullPathName(INode[] inodes, int pos) {
1826 StringBuilder fullPathName = new StringBuilder();
1827 if (inodes[0].isRoot()) {
1828 if (pos == 0) return Path.SEPARATOR;
1829 } else {
1830 fullPathName.append(inodes[0].getLocalName());
1831 }
1832
1833 for (int i=1; i<=pos; i++) {
1834 fullPathName.append(Path.SEPARATOR_CHAR).append(inodes[i].getLocalName());
1835 }
1836 return fullPathName.toString();
1837 }
1838
1839 /**
1840 * @return the relative path of an inode from one of its ancestors,
1841 * represented by an array of inodes.
1842 */
1843 private static INode[] getRelativePathINodes(INode inode, INode ancestor) {
1844 // calculate the depth of this inode from the ancestor
1845 int depth = 0;
1846 for (INode i = inode; i != null && !i.equals(ancestor); i = i.getParent()) {
1847 depth++;
1848 }
1849 INode[] inodes = new INode[depth];
1850
1851 // fill up the inodes in the path from this inode to root
1852 for (int i = 0; i < depth; i++) {
1853 if (inode == null) {
1854 NameNode.stateChangeLog.warn("Could not get full path."
1855 + " Corresponding file might have deleted already.");
1856 return null;
1857 }
1858 inodes[depth-i-1] = inode;
1859 inode = inode.getParent();
1860 }
1861 return inodes;
1862 }
1863
1864 private static INode[] getFullPathINodes(INode inode) {
1865 return getRelativePathINodes(inode, null);
1866 }
1867
1868 /** Return the full path name of the specified inode */
1869 static String getFullPathName(INode inode) {
1870 INode[] inodes = getFullPathINodes(inode);
1871 return getFullPathName(inodes, inodes.length - 1);
1872 }
1873
1874 /**
1875 * Create a directory
1876 * If ancestor directories do not exist, automatically create them.
1877
1878 * @param src string representation of the path to the directory
1879 * @param permissions the permission of the directory
1880 * @param isAutocreate if the permission of the directory should inherit
1881 * from its parent or not. u+wx is implicitly added to
1882 * the automatically created directories, and to the
1883 * given directory if inheritPermission is true
1884 * @param now creation time
1885 * @return true if the operation succeeds false otherwise
1886 * @throws FileNotFoundException if an ancestor or itself is a file
1887 * @throws QuotaExceededException if directory creation violates
1888 * any quota limit
1889 * @throws UnresolvedLinkException if a symlink is encountered in src.
1890 * @throws SnapshotAccessControlException if path is in RO snapshot
1891 */
1892 boolean mkdirs(String src, PermissionStatus permissions,
1893 boolean inheritPermission, long now)
1894 throws FileAlreadyExistsException, QuotaExceededException,
1895 UnresolvedLinkException, SnapshotAccessControlException {
1896 src = normalizePath(src);
1897 String[] names = INode.getPathNames(src);
1898 byte[][] components = INode.getPathComponents(names);
1899 final int lastInodeIndex = components.length - 1;
1900
1901 writeLock();
1902 try {
1903 INodesInPath iip = getExistingPathINodes(components);
1904 if (iip.isSnapshot()) {
1905 throw new SnapshotAccessControlException(
1906 "Modification on RO snapshot is disallowed");
1907 }
1908 INode[] inodes = iip.getINodes();
1909
1910 // find the index of the first null in inodes[]
1911 StringBuilder pathbuilder = new StringBuilder();
1912 int i = 1;
1913 for(; i < inodes.length && inodes[i] != null; i++) {
1914 pathbuilder.append(Path.SEPARATOR).append(names[i]);
1915 if (!inodes[i].isDirectory()) {
1916 throw new FileAlreadyExistsException("Parent path is not a directory: "
1917 + pathbuilder+ " "+inodes[i].getLocalName());
1918 }
1919 }
1920
1921 // default to creating parent dirs with the given perms
1922 PermissionStatus parentPermissions = permissions;
1923
1924 // if not inheriting and it's the last inode, there's no use in
1925 // computing perms that won't be used
1926 if (inheritPermission || (i < lastInodeIndex)) {
1927 // if inheriting (ie. creating a file or symlink), use the parent dir,
1928 // else the supplied permissions
1929 // NOTE: the permissions of the auto-created directories violate posix
1930 FsPermission parentFsPerm = inheritPermission
1931 ? inodes[i-1].getFsPermission() : permissions.getPermission();
1932
1933 // ensure that the permissions allow user write+execute
1934 if (!parentFsPerm.getUserAction().implies(FsAction.WRITE_EXECUTE)) {
1935 parentFsPerm = new FsPermission(
1936 parentFsPerm.getUserAction().or(FsAction.WRITE_EXECUTE),
1937 parentFsPerm.getGroupAction(),
1938 parentFsPerm.getOtherAction()
1939 );
1940 }
1941
1942 if (!parentPermissions.getPermission().equals(parentFsPerm)) {
1943 parentPermissions = new PermissionStatus(
1944 parentPermissions.getUserName(),
1945 parentPermissions.getGroupName(),
1946 parentFsPerm
1947 );
1948 // when inheriting, use same perms for entire path
1949 if (inheritPermission) permissions = parentPermissions;
1950 }
1951 }
1952
1953 // create directories beginning from the first null index
1954 for(; i < inodes.length; i++) {
1955 pathbuilder.append(Path.SEPARATOR + names[i]);
1956 unprotectedMkdir(namesystem.allocateNewInodeId(), iip, i,
1957 components[i], (i < lastInodeIndex) ? parentPermissions
1958 : permissions, now);
1959 if (inodes[i] == null) {
1960 return false;
1961 }
1962 // Directory creation also count towards FilesCreated
1963 // to match count of FilesDeleted metric.
1964 if (getFSNamesystem() != null)
1965 NameNode.getNameNodeMetrics().incrFilesCreated();
1966
1967 final String cur = pathbuilder.toString();
1968 fsImage.getEditLog().logMkDir(cur, inodes[i]);
1969 if(NameNode.stateChangeLog.isDebugEnabled()) {
1970 NameNode.stateChangeLog.debug(
1971 "DIR* FSDirectory.mkdirs: created directory " + cur);
1972 }
1973 }
1974 } finally {
1975 writeUnlock();
1976 }
1977 return true;
1978 }
1979
1980 INode unprotectedMkdir(long inodeId, String src, PermissionStatus permissions,
1981 long timestamp) throws QuotaExceededException,
1982 UnresolvedLinkException {
1983 assert hasWriteLock();
1984 byte[][] components = INode.getPathComponents(src);
1985 INodesInPath iip = getExistingPathINodes(components);
1986 INode[] inodes = iip.getINodes();
1987 final int pos = inodes.length - 1;
1988 unprotectedMkdir(inodeId, iip, pos, components[pos], permissions,
1989 timestamp);
1990 return inodes[pos];
1991 }
1992
1993 /** create a directory at index pos.
1994 * The parent path to the directory is at [0, pos-1].
1995 * All ancestors exist. Newly created one stored at index pos.
1996 */
1997 private void unprotectedMkdir(long inodeId, INodesInPath inodesInPath,
1998 int pos, byte[] name, PermissionStatus permission, long timestamp)
1999 throws QuotaExceededException {
2000 assert hasWriteLock();
2001 final INodeDirectory dir = new INodeDirectory(inodeId, name, permission,
2002 timestamp);
2003 if (addChild(inodesInPath, pos, dir, true)) {
2004 inodesInPath.setINode(pos, dir);
2005 }
2006 }
2007
2008 /**
2009 * Add the given child to the namespace.
2010 * @param src The full path name of the child node.
2011 * @throw QuotaExceededException is thrown if it violates quota limit
2012 */
2013 private boolean addINode(String src, INode child
2014 ) throws QuotaExceededException, UnresolvedLinkException {
2015 byte[][] components = INode.getPathComponents(src);
2016 child.setLocalName(components[components.length-1]);
2017 cacheName(child);
2018 writeLock();
2019 try {
2020 return addLastINode(getExistingPathINodes(components), child, true);
2021 } finally {
2022 writeUnlock();
2023 }
2024 }
2025
2026 /**
2027 * Verify quota for adding or moving a new INode with required
2028 * namespace and diskspace to a given position.
2029 *
2030 * @param inodes INodes corresponding to a path
2031 * @param pos position where a new INode will be added
2032 * @param nsDelta needed namespace
2033 * @param dsDelta needed diskspace
2034 * @param commonAncestor Last node in inodes array that is a common ancestor
2035 * for a INode that is being moved from one location to the other.
2036 * Pass null if a node is not being moved.
2037 * @throws QuotaExceededException if quota limit is exceeded.
2038 */
2039 private static void verifyQuota(INode[] inodes, int pos, long nsDelta,
2040 long dsDelta, INode commonAncestor) throws QuotaExceededException {
2041 if (nsDelta <= 0 && dsDelta <= 0) {
2042 // if quota is being freed or not being consumed
2043 return;
2044 }
2045
2046 // check existing components in the path
2047 for(int i = (pos > inodes.length? inodes.length: pos) - 1; i >= 0; i--) {
2048 if (commonAncestor == inodes[i]) {
2049 // Stop checking for quota when common ancestor is reached
2050 return;
2051 }
2052 if (inodes[i].isQuotaSet()) { // a directory with quota
2053 try {
2054 ((INodeDirectoryWithQuota) inodes[i].asDirectory()).verifyQuota(
2055 nsDelta, dsDelta);
2056 } catch (QuotaExceededException e) {
2057 e.setPathName(getFullPathName(inodes, i));
2058 throw e;
2059 }
2060 }
2061 }
2062 }
2063
2064 /**
2065 * Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
2066 * dstInodes[dstInodes.length-1]
2067 *
2068 * @param src directory from where node is being moved.
2069 * @param dst directory to where node is moved to.
2070 * @throws QuotaExceededException if quota limit is exceeded.
2071 */
2072 private void verifyQuotaForRename(INode[] src, INode[] dst)
2073 throws QuotaExceededException {
2074 if (!ready) {
2075 // Do not check quota if edits log is still being processed
2076 return;
2077 }
2078 int i = 0;
2079 for(; src[i] == dst[i]; i++);
2080 // src[i - 1] is the last common ancestor.
2081
2082 final Quota.Counts delta = src[src.length - 1].computeQuotaUsage();
2083
2084 // Reduce the required quota by dst that is being removed
2085 final int dstIndex = dst.length - 1;
2086 if (dst[dstIndex] != null) {
2087 delta.subtract(dst[dstIndex].computeQuotaUsage());
2088 }
2089 verifyQuota(dst, dstIndex, delta.get(Quota.NAMESPACE),
2090 delta.get(Quota.DISKSPACE), src[i - 1]);
2091 }
2092
2093 /** Verify if the snapshot name is legal. */
2094 void verifySnapshotName(String snapshotName, String path)
2095 throws PathComponentTooLongException {
2096 if (snapshotName.contains(Path.SEPARATOR)) {
2097 throw new HadoopIllegalArgumentException(
2098 "Snapshot name cannot contain \"" + Path.SEPARATOR + "\"");
2099 }
2100 final byte[] bytes = DFSUtil.string2Bytes(snapshotName);
2101 verifyINodeName(bytes);
2102 verifyMaxComponentLength(bytes, path, 0);
2103 }
2104
2105 /** Verify if the inode name is legal. */
2106 void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
2107 if (Arrays.equals(HdfsConstants.DOT_SNAPSHOT_DIR_BYTES, childName)) {
2108 String s = "\"" + HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name.";
2109 if (!ready) {
2110 s += " Please rename it before upgrade.";
2111 }
2112 throw new HadoopIllegalArgumentException(s);
2113 }
2114 }
2115
2116 /**
2117 * Verify child's name for fs limit.
2118 * @throws PathComponentTooLongException child's name is too long.
2119 */
2120 void verifyMaxComponentLength(byte[] childName, Object parentPath, int pos)
2121 throws PathComponentTooLongException {
2122 if (maxComponentLength == 0) {
2123 return;
2124 }
2125
2126 final int length = childName.length;
2127 if (length > maxComponentLength) {
2128 final String p = parentPath instanceof INode[]?
2129 getFullPathName((INode[])parentPath, pos - 1): (String)parentPath;
2130 final PathComponentTooLongException e = new PathComponentTooLongException(
2131 maxComponentLength, length, p, DFSUtil.bytes2String(childName));
2132 if (ready) {
2133 throw e;
2134 } else {
2135 // Do not throw if edits log is still being processed
2136 NameNode.LOG.error("ERROR in FSDirectory.verifyINodeName", e);
2137 }
2138 }
2139 }
2140
2141 /**
2142 * Verify children size for fs limit.
2143 * @throws MaxDirectoryItemsExceededException too many children.
2144 */
2145 void verifyMaxDirItems(INode[] pathComponents, int pos)
2146 throws MaxDirectoryItemsExceededException {
2147 if (maxDirItems == 0) {
2148 return;
2149 }
2150
2151 final INodeDirectory parent = pathComponents[pos-1].asDirectory();
2152 final int count = parent.getChildrenList(null).size();
2153 if (count >= maxDirItems) {
2154 final MaxDirectoryItemsExceededException e
2155 = new MaxDirectoryItemsExceededException(maxDirItems, count);
2156 if (ready) {
2157 e.setPathName(getFullPathName(pathComponents, pos - 1));
2158 throw e;
2159 } else {
2160 // Do not throw if edits log is still being processed
2161 NameNode.LOG.error("FSDirectory.verifyMaxDirItems: "
2162 + e.getLocalizedMessage());
2163 }
2164 }
2165 }
2166
2167 /**
2168 * The same as {@link #addChild(INodesInPath, int, INode, boolean)}
2169 * with pos = length - 1.
2170 */
2171 private boolean addLastINode(INodesInPath inodesInPath,
2172 INode inode, boolean checkQuota) throws QuotaExceededException {
2173 final int pos = inodesInPath.getINodes().length - 1;
2174 return addChild(inodesInPath, pos, inode, checkQuota);
2175 }
2176
2177 /** Add a node child to the inodes at index pos.
2178 * Its ancestors are stored at [0, pos-1].
2179 * @return false if the child with this name already exists;
2180 * otherwise return true;
2181 * @throw QuotaExceededException is thrown if it violates quota limit
2182 */
2183 private boolean addChild(INodesInPath iip, int pos,
2184 INode child, boolean checkQuota) throws QuotaExceededException {
2185 final INode[] inodes = iip.getINodes();
2186 // Disallow creation of /.reserved. This may be created when loading
2187 // editlog/fsimage during upgrade since /.reserved was a valid name in older
2188 // release. This may also be called when a user tries to create a file
2189 // or directory /.reserved.
2190 if (pos == 1 && inodes[0] == rootDir && isReservedName(child)) {
2191 throw new HadoopIllegalArgumentException(
2192 "File name \"" + child.getLocalName() + "\" is reserved and cannot "
2193 + "be created. If this is during upgrade change the name of the "
2194 + "existing file or directory to another name before upgrading "
2195 + "to the new release.");
2196 }
2197 // The filesystem limits are not really quotas, so this check may appear
2198 // odd. It's because a rename operation deletes the src, tries to add
2199 // to the dest, if that fails, re-adds the src from whence it came.
2200 // The rename code disables the quota when it's restoring to the
2201 // original location becase a quota violation would cause the the item
2202 // to go "poof". The fs limits must be bypassed for the same reason.
2203 if (checkQuota) {
2204 verifyMaxComponentLength(child.getLocalNameBytes(), inodes, pos);
2205 verifyMaxDirItems(inodes, pos);
2206 }
2207 // always verify inode name
2208 verifyINodeName(child.getLocalNameBytes());
2209
2210 final Quota.Counts counts = child.computeQuotaUsage();
2211 updateCount(iip, pos,
2212 counts.get(Quota.NAMESPACE), counts.get(Quota.DISKSPACE), checkQuota);
2213 final INodeDirectory parent = inodes[pos-1].asDirectory();
2214 boolean added = false;
2215 try {
2216 added = parent.addChild(child, true, iip.getLatestSnapshot(),
2217 inodeMap);
2218 } catch (QuotaExceededException e) {
2219 updateCountNoQuotaCheck(iip, pos,
2220 -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
2221 throw e;
2222 }
2223 if (!added) {
2224 updateCountNoQuotaCheck(iip, pos,
2225 -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
2226 } else {
2227 iip.setINode(pos - 1, child.getParent());
2228 addToInodeMap(child);
2229 }
2230 return added;
2231 }
2232
2233 private boolean addLastINodeNoQuotaCheck(INodesInPath inodesInPath, INode i) {
2234 try {
2235 return addLastINode(inodesInPath, i, false);
2236 } catch (QuotaExceededException e) {
2237 NameNode.LOG.warn("FSDirectory.addChildNoQuotaCheck - unexpected", e);
2238 }
2239 return false;
2240 }
2241
2242 /**
2243 * Remove the last inode in the path from the namespace.
2244 * Count of each ancestor with quota is also updated.
2245 * @return -1 for failing to remove;
2246 * 0 for removing a reference whose referred inode has other
2247 * reference nodes;
2248 * >0 otherwise.
2249 */
2250 private long removeLastINode(final INodesInPath iip)
2251 throws QuotaExceededException {
2252 final Snapshot latestSnapshot = iip.getLatestSnapshot();
2253 final INode last = iip.getLastINode();
2254 final INodeDirectory parent = iip.getINode(-2).asDirectory();
2255 if (!parent.removeChild(last, latestSnapshot, inodeMap)) {
2256 return -1;
2257 }
2258 INodeDirectory newParent = last.getParent();
2259 if (parent != newParent) {
2260 iip.setINode(-2, newParent);
2261 }
2262
2263 if (!last.isInLatestSnapshot(latestSnapshot)) {
2264 final Quota.Counts counts = last.computeQuotaUsage();
2265 updateCountNoQuotaCheck(iip, iip.getINodes().length - 1,
2266 -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
2267
2268 if (INodeReference.tryRemoveReference(last) > 0) {
2269 return 0;
2270 } else {
2271 return counts.get(Quota.NAMESPACE);
2272 }
2273 }
2274 return 1;
2275 }
2276
2277 /**
2278 */
2279 String normalizePath(String src) {
2280 if (src.length() > 1 && src.endsWith("/")) {
2281 src = src.substring(0, src.length() - 1);
2282 }
2283 return src;
2284 }
2285
2286 ContentSummary getContentSummary(String src)
2287 throws FileNotFoundException, UnresolvedLinkException {
2288 String srcs = normalizePath(src);
2289 readLock();
2290 try {
2291 INode targetNode = rootDir.getNode(srcs, false);
2292 if (targetNode == null) {
2293 throw new FileNotFoundException("File does not exist: " + srcs);
2294 }
2295 else {
2296 return targetNode.computeContentSummary();
2297 }
2298 } finally {
2299 readUnlock();
2300 }
2301 }
2302
2303 public INodeMap getINodeMap() {
2304 return inodeMap;
2305 }
2306
2307 /**
2308 * This method is always called with writeLock of FSDirectory held.
2309 */
2310 public final void addToInodeMap(INode inode) {
2311 if (inode instanceof INodeWithAdditionalFields) {
2312 inodeMap.put((INodeWithAdditionalFields)inode);
2313 }
2314 }
2315
2316
2317 /**
2318 * This method is always called with writeLock of FSDirectory held.
2319 */
2320 public final void removeFromInodeMap(List<? extends INode> inodes) {
2321 if (inodes != null) {
2322 for (INode inode : inodes) {
2323 if (inode != null && inode instanceof INodeWithAdditionalFields) {
2324 inodeMap.remove(inode);
2325 }
2326 }
2327 }
2328 }
2329
2330 /**
2331 * Get the inode from inodeMap based on its inode id.
2332 * @param id The given id
2333 * @return The inode associated with the given id
2334 */
2335 public INode getInode(long id) {
2336 readLock();
2337 try {
2338 return inodeMap.get(id);
2339 } finally {
2340 readUnlock();
2341 }
2342 }
2343
2344 @VisibleForTesting
2345 int getInodeMapSize() {
2346 return inodeMap.size();
2347 }
2348
2349 /**
2350 * See {@link ClientProtocol#setQuota(String, long, long)} for the contract.
2351 * Sets quota for for a directory.
2352 * @returns INodeDirectory if any of the quotas have changed. null other wise.
2353 * @throws FileNotFoundException if the path does not exist.
2354 * @throws PathIsNotDirectoryException if the path is not a directory.
2355 * @throws QuotaExceededException if the directory tree size is
2356 * greater than the given quota
2357 * @throws UnresolvedLinkException if a symlink is encountered in src.
2358 * @throws SnapshotAccessControlException if path is in RO snapshot
2359 */
2360 INodeDirectory unprotectedSetQuota(String src, long nsQuota, long dsQuota)
2361 throws FileNotFoundException, PathIsNotDirectoryException,
2362 QuotaExceededException, UnresolvedLinkException,
2363 SnapshotAccessControlException {
2364 assert hasWriteLock();
2365 // sanity check
2366 if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
2367 nsQuota < HdfsConstants.QUOTA_RESET) ||
2368 (dsQuota < 0 && dsQuota != HdfsConstants.QUOTA_DONT_SET &&
2369 dsQuota < HdfsConstants.QUOTA_RESET)) {
2370 throw new IllegalArgumentException("Illegal value for nsQuota or " +
2371 "dsQuota : " + nsQuota + " and " +
2372 dsQuota);
2373 }
2374
2375 String srcs = normalizePath(src);
2376 final INodesInPath iip = rootDir.getINodesInPath4Write(srcs, true);
2377 INodeDirectory dirNode = INodeDirectory.valueOf(iip.getLastINode(), srcs);
2378 if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
2379 throw new IllegalArgumentException("Cannot clear namespace quota on root.");
2380 } else { // a directory inode
2381 long oldNsQuota = dirNode.getNsQuota();
2382 long oldDsQuota = dirNode.getDsQuota();
2383 if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
2384 nsQuota = oldNsQuota;
2385 }
2386 if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
2387 dsQuota = oldDsQuota;
2388 }
2389
2390 final Snapshot latest = iip.getLatestSnapshot();
2391 if (dirNode instanceof INodeDirectoryWithQuota) {
2392 INodeDirectoryWithQuota quotaNode = (INodeDirectoryWithQuota) dirNode;
2393 Quota.Counts counts = null;
2394 if (!quotaNode.isQuotaSet()) {
2395 // dirNode must be an INodeDirectoryWithSnapshot whose quota has not
2396 // been set yet
2397 counts = quotaNode.computeQuotaUsage();
2398 }
2399 // a directory with quota; so set the quota to the new value
2400 quotaNode.setQuota(nsQuota, dsQuota);
2401 if (quotaNode.isQuotaSet() && counts != null) {
2402 quotaNode.setSpaceConsumed(counts.get(Quota.NAMESPACE),
2403 counts.get(Quota.DISKSPACE));
2404 } else if (!quotaNode.isQuotaSet() && latest == null) {
2405 // do not replace the node if the node is a snapshottable directory
2406 // without snapshots
2407 if (!(quotaNode instanceof INodeDirectoryWithSnapshot)) {
2408 // will not come here for root because root is snapshottable and
2409 // root's nsQuota is always set
2410 return quotaNode.replaceSelf4INodeDirectory(inodeMap);
2411 }
2412 }
2413 } else {
2414 // a non-quota directory; so replace it with a directory with quota
2415 return dirNode.replaceSelf4Quota(latest, nsQuota, dsQuota, inodeMap);
2416 }
2417 return (oldNsQuota != nsQuota || oldDsQuota != dsQuota) ? dirNode : null;
2418 }
2419 }
2420
2421 /**
2422 * See {@link ClientProtocol#setQuota(String, long, long)} for the contract.
2423 * @throws SnapshotAccessControlException if path is in RO snapshot
2424 * @see #unprotectedSetQuota(String, long, long)
2425 */
2426 void setQuota(String src, long nsQuota, long dsQuota)
2427 throws FileNotFoundException, PathIsNotDirectoryException,
2428 QuotaExceededException, UnresolvedLinkException,
2429 SnapshotAccessControlException {
2430 writeLock();
2431 try {
2432 INodeDirectory dir = unprotectedSetQuota(src, nsQuota, dsQuota);
2433 if (dir != null) {
2434 fsImage.getEditLog().logSetQuota(src, dir.getNsQuota(),
2435 dir.getDsQuota());
2436 }
2437 } finally {
2438 writeUnlock();
2439 }
2440 }
2441
2442 long totalInodes() {
2443 readLock();
2444 try {
2445 return rootDir.numItemsInTree();
2446 } finally {
2447 readUnlock();
2448 }
2449 }
2450
2451 /**
2452 * Sets the access time on the file/directory. Logs it in the transaction log.
2453 */
2454 void setTimes(String src, INode inode, long mtime, long atime, boolean force,
2455 Snapshot latest) throws QuotaExceededException {
2456 boolean status = false;
2457 writeLock();
2458 try {
2459 status = unprotectedSetTimes(inode, mtime, atime, force, latest);
2460 } finally {
2461 writeUnlock();
2462 }
2463 if (status) {
2464 fsImage.getEditLog().logTimes(src, mtime, atime);
2465 }
2466 }
2467
2468 boolean unprotectedSetTimes(String src, long mtime, long atime, boolean force)
2469 throws UnresolvedLinkException, QuotaExceededException {
2470 assert hasWriteLock();
2471 final INodesInPath i = getLastINodeInPath(src);
2472 return unprotectedSetTimes(i.getLastINode(), mtime, atime, force,
2473 i.getLatestSnapshot());
2474 }
2475
2476 private boolean unprotectedSetTimes(INode inode, long mtime,
2477 long atime, boolean force, Snapshot latest) throws QuotaExceededException {
2478 assert hasWriteLock();
2479 boolean status = false;
2480 if (mtime != -1) {
2481 inode = inode.setModificationTime(mtime, latest, inodeMap);
2482 status = true;
2483 }
2484 if (atime != -1) {
2485 long inodeTime = inode.getAccessTime(null);
2486
2487 // if the last access time update was within the last precision interval, then
2488 // no need to store access time
2489 if (atime <= inodeTime + getFSNamesystem().getAccessTimePrecision() && !force) {
2490 status = false;
2491 } else {
2492 inode.setAccessTime(atime, latest, inodeMap);
2493 status = true;
2494 }
2495 }
2496 return status;
2497 }
2498
2499 /**
2500 * Reset the entire namespace tree.
2501 */
2502 void reset() {
2503 writeLock();
2504 try {
2505 setReady(false);
2506 rootDir = createRoot(getFSNamesystem());
2507 inodeMap.clear();
2508 addToInodeMap(rootDir);
2509 nameCache.reset();
2510 } finally {
2511 writeUnlock();
2512 }
2513 }
2514
2515 /**
2516 * create an hdfs file status from an inode
2517 *
2518 * @param path the local name
2519 * @param node inode
2520 * @param needLocation if block locations need to be included or not
2521 * @return a file status
2522 * @throws IOException if any error occurs
2523 */
2524 private HdfsFileStatus createFileStatus(byte[] path, INode node,
2525 boolean needLocation, Snapshot snapshot) throws IOException {
2526 if (needLocation) {
2527 return createLocatedFileStatus(path, node, snapshot);
2528 } else {
2529 return createFileStatus(path, node, snapshot);
2530 }
2531 }
2532 /**
2533 * Create FileStatus by file INode
2534 */
2535 HdfsFileStatus createFileStatus(byte[] path, INode node,
2536 Snapshot snapshot) {
2537 long size = 0; // length is zero for directories
2538 short replication = 0;
2539 long blocksize = 0;
2540 if (node.isFile()) {
2541 final INodeFile fileNode = node.asFile();
2542 size = fileNode.computeFileSize(snapshot);
2543 replication = fileNode.getFileReplication(snapshot);
2544 blocksize = fileNode.getPreferredBlockSize();
2545 }
2546 int childrenNum = node.isDirectory() ?
2547 node.asDirectory().getChildrenNum(snapshot) : 0;
2548
2549 return new HdfsFileStatus(
2550 size,
2551 node.isDirectory(),
2552 replication,
2553 blocksize,
2554 node.getModificationTime(snapshot),
2555 node.getAccessTime(snapshot),
2556 node.getFsPermission(snapshot),
2557 node.getUserName(snapshot),
2558 node.getGroupName(snapshot),
2559 node.isSymlink() ? node.asSymlink().getSymlink() : null,
2560 path,
2561 node.getId(),
2562 childrenNum);
2563 }
2564
2565 /**
2566 * Create FileStatus with location info by file INode
2567 */
2568 private HdfsLocatedFileStatus createLocatedFileStatus(byte[] path,
2569 INode node, Snapshot snapshot) throws IOException {
2570 assert hasReadLock();
2571 long size = 0; // length is zero for directories
2572 short replication = 0;
2573 long blocksize = 0;
2574 LocatedBlocks loc = null;
2575 if (node.isFile()) {
2576 final INodeFile fileNode = node.asFile();
2577 size = fileNode.computeFileSize(snapshot);
2578 replication = fileNode.getFileReplication(snapshot);
2579 blocksize = fileNode.getPreferredBlockSize();
2580
2581 final boolean inSnapshot = snapshot != null;
2582 final boolean isUc = inSnapshot ? false : fileNode.isUnderConstruction();
2583 final long fileSize = !inSnapshot && isUc ?
2584 fileNode.computeFileSizeNotIncludingLastUcBlock() : size;
2585 loc = getFSNamesystem().getBlockManager().createLocatedBlocks(
2586 fileNode.getBlocks(), fileSize, isUc, 0L, size, false,
2587 inSnapshot);
2588 if (loc == null) {
2589 loc = new LocatedBlocks();
2590 }
2591 }
2592 int childrenNum = node.isDirectory() ?
2593 node.asDirectory().getChildrenNum(snapshot) : 0;
2594
2595 return new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
2596 blocksize, node.getModificationTime(snapshot),
2597 node.getAccessTime(snapshot), node.getFsPermission(snapshot),
2598 node.getUserName(snapshot), node.getGroupName(snapshot),
2599 node.isSymlink() ? node.asSymlink().getSymlink() : null, path,
2600 node.getId(), loc, childrenNum);
2601 }
2602
2603
2604 /**
2605 * Add the given symbolic link to the fs. Record it in the edits log.
2606 */
2607 INodeSymlink addSymlink(String path, String target,
2608 PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
2609 throws UnresolvedLinkException, FileAlreadyExistsException,
2610 QuotaExceededException, SnapshotAccessControlException {
2611 waitForReady();
2612
2613 final long modTime = now();
2614 if (createParent) {
2615 final String parent = new Path(path).getParent().toString();
2616 if (!mkdirs(parent, dirPerms, true, modTime)) {
2617 return null;
2618 }
2619 }
2620 final String userName = dirPerms.getUserName();
2621 INodeSymlink newNode = null;
2622 long id = namesystem.allocateNewInodeId();
2623 writeLock();
2624 try {
2625 newNode = unprotectedAddSymlink(id, path, target, modTime, modTime,
2626 new PermissionStatus(userName, null, FsPermission.getDefault()));
2627 } finally {
2628 writeUnlock();
2629 }
2630 if (newNode == null) {
2631 NameNode.stateChangeLog.info("DIR* addSymlink: failed to add " + path);
2632 return null;
2633 }
2634 fsImage.getEditLog().logSymlink(path, target, modTime, modTime, newNode,
2635 logRetryCache);
2636
2637 if(NameNode.stateChangeLog.isDebugEnabled()) {
2638 NameNode.stateChangeLog.debug("DIR* addSymlink: " + path + " is added");
2639 }
2640 return newNode;
2641 }
2642
2643 /**
2644 * Add the specified path into the namespace. Invoked from edit log processing.
2645 */
2646 INodeSymlink unprotectedAddSymlink(long id, String path, String target,
2647 long mtime, long atime, PermissionStatus perm)
2648 throws UnresolvedLinkException, QuotaExceededException {
2649 assert hasWriteLock();
2650 final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
2651 target);
2652 return addINode(path, symlink) ? symlink : null;
2653 }
2654
2655 /**
2656 * Caches frequently used file names to reuse file name objects and
2657 * reduce heap size.
2658 */
2659 void cacheName(INode inode) {
2660 // Name is cached only for files
2661 if (!inode.isFile()) {
2662 return;
2663 }
2664 ByteArray name = new ByteArray(inode.getLocalNameBytes());
2665 name = nameCache.put(name);
2666 if (name != null) {
2667 inode.setLocalName(name.getBytes());
2668 }
2669 }
2670
2671 void shutdown() {
2672 nameCache.reset();
2673 inodeMap.clear();
2674 }
2675
2676 /**
2677 * Given an INode get all the path complents leading to it from the root.
2678 * If an Inode corresponding to C is given in /A/B/C, the returned
2679 * patch components will be {root, A, B, C}
2680 */
2681 static byte[][] getPathComponents(INode inode) {
2682 List<byte[]> components = new ArrayList<byte[]>();
2683 components.add(0, inode.getLocalNameBytes());
2684 while(inode.getParent() != null) {
2685 components.add(0, inode.getParent().getLocalNameBytes());
2686 inode = inode.getParent();
2687 }
2688 return components.toArray(new byte[components.size()][]);
2689 }
2690
2691 /**
2692 * @return path components for reserved path, else null.
2693 */
2694 static byte[][] getPathComponentsForReservedPath(String src) {
2695 return !isReservedName(src) ? null : INode.getPathComponents(src);
2696 }
2697
2698 /**
2699 * Resolve the path of /.reserved/.inodes/<inodeid>/... to a regular path
2700 *
2701 * @param src path that is being processed
2702 * @param pathComponents path components corresponding to the path
2703 * @param fsd FSDirectory
2704 * @return if the path indicates an inode, return path after replacing upto
2705 * <inodeid> with the corresponding path of the inode, else the path
2706 * in {@code src} as is.
2707 * @throws FileNotFoundException if inodeid is invalid
2708 */
2709 static String resolvePath(String src, byte[][] pathComponents, FSDirectory fsd)
2710 throws FileNotFoundException {
2711 if (pathComponents == null || pathComponents.length <= 3) {
2712 return src;
2713 }
2714 // Not /.reserved/.inodes
2715 if (!Arrays.equals(DOT_RESERVED, pathComponents[1])
2716 || !Arrays.equals(DOT_INODES, pathComponents[2])) { // Not .inodes path
2717 return src;
2718 }
2719 final String inodeId = DFSUtil.bytes2String(pathComponents[3]);
2720 long id = 0;
2721 try {
2722 id = Long.valueOf(inodeId);
2723 } catch (NumberFormatException e) {
2724 throw new FileNotFoundException("Invalid inode path: " + src);
2725 }
2726 if (id == INodeId.ROOT_INODE_ID && pathComponents.length == 4) {
2727 return Path.SEPARATOR;
2728 }
2729 INode inode = fsd.getInode(id);
2730 if (inode == null) {
2731 throw new FileNotFoundException(
2732 "File for given inode path does not exist: " + src);
2733 }
2734
2735 // Handle single ".." for NFS lookup support.
2736 if ((pathComponents.length > 4)
2737 && DFSUtil.bytes2String(pathComponents[4]).equals("..")) {
2738 INode parent = inode.getParent();
2739 if (parent == null || parent.getId() == INodeId.ROOT_INODE_ID) {
2740 // inode is root, or its parent is root.
2741 return Path.SEPARATOR;
2742 } else {
2743 return parent.getFullPathName();
2744 }
2745 }
2746
2747 StringBuilder path = id == INodeId.ROOT_INODE_ID ? new StringBuilder()
2748 : new StringBuilder(inode.getFullPathName());
2749 for (int i = 4; i < pathComponents.length; i++) {
2750 path.append(Path.SEPARATOR).append(DFSUtil.bytes2String(pathComponents[i]));
2751 }
2752 if (NameNode.LOG.isDebugEnabled()) {
2753 NameNode.LOG.debug("Resolved path is " + path);
2754 }
2755 return path.toString();
2756 }
2757
2758 /** Check if a given inode name is reserved */
2759 public static boolean isReservedName(INode inode) {
2760 return CHECK_RESERVED_FILE_NAMES
2761 && Arrays.equals(inode.getLocalNameBytes(), DOT_RESERVED);
2762 }
2763
2764 /** Check if a given path is reserved */
2765 public static boolean isReservedName(String src) {
2766 return src.startsWith(DOT_RESERVED_PATH_PREFIX);
2767 }
2768 }