001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.namenode.snapshot;
019
020 import java.io.IOException;
021 import java.io.PrintWriter;
022 import java.util.ArrayList;
023 import java.util.Collections;
024 import java.util.Comparator;
025 import java.util.HashMap;
026 import java.util.Iterator;
027 import java.util.List;
028 import java.util.Map;
029 import java.util.SortedMap;
030 import java.util.TreeMap;
031
032 import org.apache.hadoop.HadoopIllegalArgumentException;
033 import org.apache.hadoop.classification.InterfaceAudience;
034 import org.apache.hadoop.hdfs.DFSUtil;
035 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
036 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
037 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
038 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
039 import org.apache.hadoop.hdfs.protocol.SnapshotException;
040 import org.apache.hadoop.hdfs.server.namenode.Content;
041 import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
042 import org.apache.hadoop.hdfs.server.namenode.INode;
043 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
044 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
045 import org.apache.hadoop.hdfs.server.namenode.INodeMap;
046 import org.apache.hadoop.hdfs.server.namenode.Quota;
047 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
048 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
049 import org.apache.hadoop.hdfs.util.Diff.ListType;
050 import org.apache.hadoop.hdfs.util.ReadOnlyList;
051 import org.apache.hadoop.util.Time;
052
053 import com.google.common.base.Preconditions;
054 import com.google.common.primitives.SignedBytes;
055
056 /**
057 * Directories where taking snapshots is allowed.
058 *
059 * Like other {@link INode} subclasses, this class is synchronized externally
060 * by the namesystem and FSDirectory locks.
061 */
062 @InterfaceAudience.Private
063 public class INodeDirectorySnapshottable extends INodeDirectory {
064 /** Limit the number of snapshot per snapshottable directory. */
065 static final int SNAPSHOT_LIMIT = 1 << 16;
066
067 /** Cast INode to INodeDirectorySnapshottable. */
068 static public INodeDirectorySnapshottable valueOf(
069 INode inode, String src) throws IOException {
070 final INodeDirectory dir = INodeDirectory.valueOf(inode, src);
071 if (!dir.isSnapshottable()) {
072 throw new SnapshotException(
073 "Directory is not a snapshottable directory: " + src);
074 }
075 return (INodeDirectorySnapshottable)dir;
076 }
077
078 /**
079 * A class describing the difference between snapshots of a snapshottable
080 * directory.
081 */
082 public static class SnapshotDiffInfo {
083 /** Compare two inodes based on their full names */
084 public static final Comparator<INode> INODE_COMPARATOR =
085 new Comparator<INode>() {
086 @Override
087 public int compare(INode left, INode right) {
088 if (left == null) {
089 return right == null ? 0 : -1;
090 } else {
091 if (right == null) {
092 return 1;
093 } else {
094 int cmp = compare(left.getParent(), right.getParent());
095 return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
096 left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
097 }
098 }
099 }
100 };
101
102 /** The root directory of the snapshots */
103 private final INodeDirectorySnapshottable snapshotRoot;
104 /** The starting point of the difference */
105 private final Snapshot from;
106 /** The end point of the difference */
107 private final Snapshot to;
108 /**
109 * A map recording modified INodeFile and INodeDirectory and their relative
110 * path corresponding to the snapshot root. Sorted based on their names.
111 */
112 private final SortedMap<INode, byte[][]> diffMap =
113 new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
114 /**
115 * A map capturing the detailed difference about file creation/deletion.
116 * Each key indicates a directory whose children have been changed between
117 * the two snapshots, while its associated value is a {@link ChildrenDiff}
118 * storing the changes (creation/deletion) happened to the children (files).
119 */
120 private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
121 new HashMap<INodeDirectory, ChildrenDiff>();
122
123 SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
124 Snapshot end) {
125 this.snapshotRoot = snapshotRoot;
126 this.from = start;
127 this.to = end;
128 }
129
130 /** Add a dir-diff pair */
131 private void addDirDiff(INodeDirectory dir, byte[][] relativePath,
132 ChildrenDiff diff) {
133 dirDiffMap.put(dir, diff);
134 diffMap.put(dir, relativePath);
135 }
136
137 /** Add a modified file */
138 private void addFileDiff(INodeFile file, byte[][] relativePath) {
139 diffMap.put(file, relativePath);
140 }
141
142 /** @return True if {@link #from} is earlier than {@link #to} */
143 private boolean isFromEarlier() {
144 return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
145 }
146
147 /**
148 * Generate a {@link SnapshotDiffReport} based on detailed diff information.
149 * @return A {@link SnapshotDiffReport} describing the difference
150 */
151 public SnapshotDiffReport generateReport() {
152 List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
153 for (INode node : diffMap.keySet()) {
154 diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
155 .get(node)));
156 if (node.isDirectory()) {
157 ChildrenDiff dirDiff = dirDiffMap.get(node);
158 List<DiffReportEntry> subList = dirDiff.generateReport(
159 diffMap.get(node), isFromEarlier());
160 diffReportList.addAll(subList);
161 }
162 }
163 return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
164 Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
165 diffReportList);
166 }
167 }
168
169 /**
170 * Snapshots of this directory in ascending order of snapshot names.
171 * Note that snapshots in ascending order of snapshot id are stored in
172 * {@link INodeDirectoryWithSnapshot}.diffs (a private field).
173 */
174 private final List<Snapshot> snapshotsByNames = new ArrayList<Snapshot>();
175
176 /**
177 * @return {@link #snapshotsByNames}
178 */
179 ReadOnlyList<Snapshot> getSnapshotsByNames() {
180 return ReadOnlyList.Util.asReadOnlyList(this.snapshotsByNames);
181 }
182
183 /** Number of snapshots allowed. */
184 private int snapshotQuota = SNAPSHOT_LIMIT;
185
186 public INodeDirectorySnapshottable(INodeDirectory dir) {
187 super(dir, true, dir.getFeatures());
188 // add snapshot feature if the original directory does not have it
189 if (!isWithSnapshot()) {
190 addSnapshotFeature(null);
191 }
192 }
193
194 /** @return the number of existing snapshots. */
195 public int getNumSnapshots() {
196 return snapshotsByNames.size();
197 }
198
199 private int searchSnapshot(byte[] snapshotName) {
200 return Collections.binarySearch(snapshotsByNames, snapshotName);
201 }
202
203 /** @return the snapshot with the given name. */
204 public Snapshot getSnapshot(byte[] snapshotName) {
205 final int i = searchSnapshot(snapshotName);
206 return i < 0? null: snapshotsByNames.get(i);
207 }
208
209 Snapshot getSnapshotById(int sid) {
210 for (Snapshot s : snapshotsByNames) {
211 if (s.getId() == sid) {
212 return s;
213 }
214 }
215 return null;
216 }
217
218 /** @return {@link #snapshotsByNames} as a {@link ReadOnlyList} */
219 public ReadOnlyList<Snapshot> getSnapshotList() {
220 return ReadOnlyList.Util.asReadOnlyList(snapshotsByNames);
221 }
222
223 /**
224 * Rename a snapshot
225 * @param path
226 * The directory path where the snapshot was taken. Used for
227 * generating exception message.
228 * @param oldName
229 * Old name of the snapshot
230 * @param newName
231 * New name the snapshot will be renamed to
232 * @throws SnapshotException
233 * Throw SnapshotException when either the snapshot with the old
234 * name does not exist or a snapshot with the new name already
235 * exists
236 */
237 public void renameSnapshot(String path, String oldName, String newName)
238 throws SnapshotException {
239 if (newName.equals(oldName)) {
240 return;
241 }
242 final int indexOfOld = searchSnapshot(DFSUtil.string2Bytes(oldName));
243 if (indexOfOld < 0) {
244 throw new SnapshotException("The snapshot " + oldName
245 + " does not exist for directory " + path);
246 } else {
247 final byte[] newNameBytes = DFSUtil.string2Bytes(newName);
248 int indexOfNew = searchSnapshot(newNameBytes);
249 if (indexOfNew > 0) {
250 throw new SnapshotException("The snapshot " + newName
251 + " already exists for directory " + path);
252 }
253 // remove the one with old name from snapshotsByNames
254 Snapshot snapshot = snapshotsByNames.remove(indexOfOld);
255 final INodeDirectory ssRoot = snapshot.getRoot();
256 ssRoot.setLocalName(newNameBytes);
257 indexOfNew = -indexOfNew - 1;
258 if (indexOfNew <= indexOfOld) {
259 snapshotsByNames.add(indexOfNew, snapshot);
260 } else { // indexOfNew > indexOfOld
261 snapshotsByNames.add(indexOfNew - 1, snapshot);
262 }
263 }
264 }
265
266 public int getSnapshotQuota() {
267 return snapshotQuota;
268 }
269
270 public void setSnapshotQuota(int snapshotQuota) {
271 if (snapshotQuota < 0) {
272 throw new HadoopIllegalArgumentException(
273 "Cannot set snapshot quota to " + snapshotQuota + " < 0");
274 }
275 this.snapshotQuota = snapshotQuota;
276 }
277
278 @Override
279 public boolean isSnapshottable() {
280 return true;
281 }
282
283 /**
284 * Simply add a snapshot into the {@link #snapshotsByNames}. Used by FSImage
285 * loading.
286 */
287 void addSnapshot(Snapshot snapshot) {
288 this.snapshotsByNames.add(snapshot);
289 }
290
291 /** Add a snapshot. */
292 Snapshot addSnapshot(int id, String name) throws SnapshotException,
293 QuotaExceededException {
294 //check snapshot quota
295 final int n = getNumSnapshots();
296 if (n + 1 > snapshotQuota) {
297 throw new SnapshotException("Failed to add snapshot: there are already "
298 + n + " snapshot(s) and the snapshot quota is "
299 + snapshotQuota);
300 }
301 final Snapshot s = new Snapshot(id, name, this);
302 final byte[] nameBytes = s.getRoot().getLocalNameBytes();
303 final int i = searchSnapshot(nameBytes);
304 if (i >= 0) {
305 throw new SnapshotException("Failed to add snapshot: there is already a "
306 + "snapshot with the same name \"" + Snapshot.getSnapshotName(s) + "\".");
307 }
308
309 final DirectoryDiff d = getDiffs().addDiff(id, this);
310 d.setSnapshotRoot(s.getRoot());
311 snapshotsByNames.add(-i - 1, s);
312
313 //set modification time
314 updateModificationTime(Time.now(), Snapshot.CURRENT_STATE_ID);
315 s.getRoot().setModificationTime(getModificationTime(),
316 Snapshot.CURRENT_STATE_ID);
317 return s;
318 }
319
320 /**
321 * Remove the snapshot with the given name from {@link #snapshotsByNames},
322 * and delete all the corresponding DirectoryDiff.
323 *
324 * @param snapshotName The name of the snapshot to be removed
325 * @param collectedBlocks Used to collect information to update blocksMap
326 * @return The removed snapshot. Null if no snapshot with the given name
327 * exists.
328 */
329 Snapshot removeSnapshot(String snapshotName,
330 BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
331 throws SnapshotException {
332 final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
333 if (i < 0) {
334 throw new SnapshotException("Cannot delete snapshot " + snapshotName
335 + " from path " + this.getFullPathName()
336 + ": the snapshot does not exist.");
337 } else {
338 final Snapshot snapshot = snapshotsByNames.get(i);
339 int prior = Snapshot.findLatestSnapshot(this, snapshot.getId());
340 try {
341 Quota.Counts counts = cleanSubtree(snapshot.getId(), prior,
342 collectedBlocks, removedINodes, true);
343 INodeDirectory parent = getParent();
344 if (parent != null) {
345 // there will not be any WithName node corresponding to the deleted
346 // snapshot, thus only update the quota usage in the current tree
347 parent.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
348 -counts.get(Quota.DISKSPACE), true);
349 }
350 } catch(QuotaExceededException e) {
351 LOG.error("BUG: removeSnapshot increases namespace usage.", e);
352 }
353 // remove from snapshotsByNames after successfully cleaning the subtree
354 snapshotsByNames.remove(i);
355 return snapshot;
356 }
357 }
358
359 @Override
360 public ContentSummaryComputationContext computeContentSummary(
361 final ContentSummaryComputationContext summary) {
362 super.computeContentSummary(summary);
363 summary.getCounts().add(Content.SNAPSHOT, snapshotsByNames.size());
364 summary.getCounts().add(Content.SNAPSHOTTABLE_DIRECTORY, 1);
365 return summary;
366 }
367
368 /**
369 * Compute the difference between two snapshots (or a snapshot and the current
370 * directory) of the directory.
371 *
372 * @param from The name of the start point of the comparison. Null indicating
373 * the current tree.
374 * @param to The name of the end point. Null indicating the current tree.
375 * @return The difference between the start/end points.
376 * @throws SnapshotException If there is no snapshot matching the starting
377 * point, or if endSnapshotName is not null but cannot be identified
378 * as a previous snapshot.
379 */
380 SnapshotDiffInfo computeDiff(final String from, final String to)
381 throws SnapshotException {
382 Snapshot fromSnapshot = getSnapshotByName(from);
383 Snapshot toSnapshot = getSnapshotByName(to);
384 // if the start point is equal to the end point, return null
385 if (from.equals(to)) {
386 return null;
387 }
388 SnapshotDiffInfo diffs = new SnapshotDiffInfo(this, fromSnapshot,
389 toSnapshot);
390 computeDiffRecursively(this, new ArrayList<byte[]>(), diffs);
391 return diffs;
392 }
393
394 /**
395 * Find the snapshot matching the given name.
396 *
397 * @param snapshotName The name of the snapshot.
398 * @return The corresponding snapshot. Null if snapshotName is null or empty.
399 * @throws SnapshotException If snapshotName is not null or empty, but there
400 * is no snapshot matching the name.
401 */
402 private Snapshot getSnapshotByName(String snapshotName)
403 throws SnapshotException {
404 Snapshot s = null;
405 if (snapshotName != null && !snapshotName.isEmpty()) {
406 final int index = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
407 if (index < 0) {
408 throw new SnapshotException("Cannot find the snapshot of directory "
409 + this.getFullPathName() + " with name " + snapshotName);
410 }
411 s = snapshotsByNames.get(index);
412 }
413 return s;
414 }
415
416 /**
417 * Recursively compute the difference between snapshots under a given
418 * directory/file.
419 * @param node The directory/file under which the diff is computed.
420 * @param parentPath Relative path (corresponding to the snapshot root) of
421 * the node's parent.
422 * @param diffReport data structure used to store the diff.
423 */
424 private void computeDiffRecursively(INode node, List<byte[]> parentPath,
425 SnapshotDiffInfo diffReport) {
426 ChildrenDiff diff = new ChildrenDiff();
427 byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
428 if (node.isDirectory()) {
429 INodeDirectory dir = node.asDirectory();
430 DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
431 if (sf != null) {
432 boolean change = sf.computeDiffBetweenSnapshots(diffReport.from,
433 diffReport.to, diff, dir);
434 if (change) {
435 diffReport.addDirDiff(dir, relativePath, diff);
436 }
437 }
438 ReadOnlyList<INode> children = dir.getChildrenList(
439 diffReport.isFromEarlier() ? Snapshot.getSnapshotId(diffReport.to) :
440 Snapshot.getSnapshotId(diffReport.from));
441 for (INode child : children) {
442 final byte[] name = child.getLocalNameBytes();
443 if (diff.searchIndex(ListType.CREATED, name) < 0
444 && diff.searchIndex(ListType.DELETED, name) < 0) {
445 parentPath.add(name);
446 computeDiffRecursively(child, parentPath, diffReport);
447 parentPath.remove(parentPath.size() - 1);
448 }
449 }
450 } else if (node.isFile() && node.asFile().isWithSnapshot()) {
451 INodeFile file = node.asFile();
452 Snapshot earlierSnapshot = diffReport.isFromEarlier() ? diffReport.from
453 : diffReport.to;
454 Snapshot laterSnapshot = diffReport.isFromEarlier() ? diffReport.to
455 : diffReport.from;
456 boolean change = file.getDiffs().changedBetweenSnapshots(earlierSnapshot,
457 laterSnapshot);
458 if (change) {
459 diffReport.addFileDiff(file, relativePath);
460 }
461 }
462 }
463
464 /**
465 * Replace itself with {@link INodeDirectoryWithSnapshot} or
466 * {@link INodeDirectory} depending on the latest snapshot.
467 */
468 INodeDirectory replaceSelf(final int latestSnapshotId, final INodeMap inodeMap)
469 throws QuotaExceededException {
470 if (latestSnapshotId == Snapshot.CURRENT_STATE_ID) {
471 Preconditions.checkState(getDirectoryWithSnapshotFeature()
472 .getLastSnapshotId() == Snapshot.CURRENT_STATE_ID, "this=%s", this);
473 }
474 INodeDirectory dir = replaceSelf4INodeDirectory(inodeMap);
475 if (latestSnapshotId != Snapshot.CURRENT_STATE_ID) {
476 dir.recordModification(latestSnapshotId);
477 }
478 return dir;
479 }
480
481 @Override
482 public String toDetailString() {
483 return super.toDetailString() + ", snapshotsByNames=" + snapshotsByNames;
484 }
485
486 @Override
487 public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
488 int snapshot) {
489 super.dumpTreeRecursively(out, prefix, snapshot);
490
491 if (snapshot == Snapshot.CURRENT_STATE_ID) {
492 out.println();
493 out.print(prefix);
494
495 out.print("Snapshot of ");
496 final String name = getLocalName();
497 out.print(name.isEmpty()? "/": name);
498 out.print(": quota=");
499 out.print(getSnapshotQuota());
500
501 int n = 0;
502 for(DirectoryDiff diff : getDiffs()) {
503 if (diff.isSnapshotRoot()) {
504 n++;
505 }
506 }
507 Preconditions.checkState(n == snapshotsByNames.size(), "#n=" + n
508 + ", snapshotsByNames.size()=" + snapshotsByNames.size());
509 out.print(", #snapshot=");
510 out.println(n);
511
512 dumpTreeRecursively(out, prefix, new Iterable<SnapshotAndINode>() {
513 @Override
514 public Iterator<SnapshotAndINode> iterator() {
515 return new Iterator<SnapshotAndINode>() {
516 final Iterator<DirectoryDiff> i = getDiffs().iterator();
517 private DirectoryDiff next = findNext();
518
519 private DirectoryDiff findNext() {
520 for(; i.hasNext(); ) {
521 final DirectoryDiff diff = i.next();
522 if (diff.isSnapshotRoot()) {
523 return diff;
524 }
525 }
526 return null;
527 }
528
529 @Override
530 public boolean hasNext() {
531 return next != null;
532 }
533
534 @Override
535 public SnapshotAndINode next() {
536 final SnapshotAndINode pair = new SnapshotAndINode(next
537 .getSnapshotId(), getSnapshotById(next.getSnapshotId())
538 .getRoot());
539 next = findNext();
540 return pair;
541 }
542
543 @Override
544 public void remove() {
545 throw new UnsupportedOperationException();
546 }
547 };
548 }
549 });
550 }
551 }
552 }