001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.namenode.snapshot;
019
020 import java.io.IOException;
021 import java.io.PrintWriter;
022 import java.util.ArrayList;
023 import java.util.Collections;
024 import java.util.Comparator;
025 import java.util.HashMap;
026 import java.util.Iterator;
027 import java.util.List;
028 import java.util.Map;
029 import java.util.SortedMap;
030 import java.util.TreeMap;
031
032 import org.apache.hadoop.HadoopIllegalArgumentException;
033 import org.apache.hadoop.classification.InterfaceAudience;
034 import org.apache.hadoop.hdfs.DFSUtil;
035 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
036 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
037 import org.apache.hadoop.hdfs.protocol.SnapshotException;
038 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
039 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
040 import org.apache.hadoop.hdfs.server.namenode.Content;
041 import org.apache.hadoop.hdfs.server.namenode.INode;
042 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
043 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
044 import org.apache.hadoop.hdfs.server.namenode.INodeMap;
045 import org.apache.hadoop.hdfs.server.namenode.Quota;
046 import org.apache.hadoop.hdfs.util.Diff.ListType;
047 import org.apache.hadoop.hdfs.util.ReadOnlyList;
048 import org.apache.hadoop.util.Time;
049
050 import com.google.common.base.Preconditions;
051 import com.google.common.primitives.SignedBytes;
052
053 /**
054 * Directories where taking snapshots is allowed.
055 *
056 * Like other {@link INode} subclasses, this class is synchronized externally
057 * by the namesystem and FSDirectory locks.
058 */
059 @InterfaceAudience.Private
060 public class INodeDirectorySnapshottable extends INodeDirectoryWithSnapshot {
061 /** Limit the number of snapshot per snapshottable directory. */
062 static final int SNAPSHOT_LIMIT = 1 << 16;
063
064 /** Cast INode to INodeDirectorySnapshottable. */
065 static public INodeDirectorySnapshottable valueOf(
066 INode inode, String src) throws IOException {
067 final INodeDirectory dir = INodeDirectory.valueOf(inode, src);
068 if (!dir.isSnapshottable()) {
069 throw new SnapshotException(
070 "Directory is not a snapshottable directory: " + src);
071 }
072 return (INodeDirectorySnapshottable)dir;
073 }
074
075 /**
076 * A class describing the difference between snapshots of a snapshottable
077 * directory.
078 */
079 public static class SnapshotDiffInfo {
080 /** Compare two inodes based on their full names */
081 public static final Comparator<INode> INODE_COMPARATOR =
082 new Comparator<INode>() {
083 @Override
084 public int compare(INode left, INode right) {
085 if (left == null) {
086 return right == null ? 0 : -1;
087 } else {
088 if (right == null) {
089 return 1;
090 } else {
091 int cmp = compare(left.getParent(), right.getParent());
092 return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
093 left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
094 }
095 }
096 }
097 };
098
099 /** The root directory of the snapshots */
100 private final INodeDirectorySnapshottable snapshotRoot;
101 /** The starting point of the difference */
102 private final Snapshot from;
103 /** The end point of the difference */
104 private final Snapshot to;
105 /**
106 * A map recording modified INodeFile and INodeDirectory and their relative
107 * path corresponding to the snapshot root. Sorted based on their names.
108 */
109 private final SortedMap<INode, byte[][]> diffMap =
110 new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
111 /**
112 * A map capturing the detailed difference about file creation/deletion.
113 * Each key indicates a directory whose children have been changed between
114 * the two snapshots, while its associated value is a {@link ChildrenDiff}
115 * storing the changes (creation/deletion) happened to the children (files).
116 */
117 private final Map<INodeDirectoryWithSnapshot, ChildrenDiff> dirDiffMap =
118 new HashMap<INodeDirectoryWithSnapshot, ChildrenDiff>();
119
120 SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
121 Snapshot end) {
122 this.snapshotRoot = snapshotRoot;
123 this.from = start;
124 this.to = end;
125 }
126
127 /** Add a dir-diff pair */
128 private void addDirDiff(INodeDirectoryWithSnapshot dir,
129 byte[][] relativePath, ChildrenDiff diff) {
130 dirDiffMap.put(dir, diff);
131 diffMap.put(dir, relativePath);
132 }
133
134 /** Add a modified file */
135 private void addFileDiff(INodeFile file, byte[][] relativePath) {
136 diffMap.put(file, relativePath);
137 }
138
139 /** @return True if {@link #from} is earlier than {@link #to} */
140 private boolean isFromEarlier() {
141 return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
142 }
143
144 /**
145 * Generate a {@link SnapshotDiffReport} based on detailed diff information.
146 * @return A {@link SnapshotDiffReport} describing the difference
147 */
148 public SnapshotDiffReport generateReport() {
149 List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
150 for (INode node : diffMap.keySet()) {
151 diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
152 .get(node)));
153 if (node.isDirectory()) {
154 ChildrenDiff dirDiff = dirDiffMap.get(node);
155 List<DiffReportEntry> subList = dirDiff.generateReport(
156 diffMap.get(node), (INodeDirectoryWithSnapshot) node,
157 isFromEarlier());
158 diffReportList.addAll(subList);
159 }
160 }
161 return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
162 Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
163 diffReportList);
164 }
165 }
166
167 /**
168 * Snapshots of this directory in ascending order of snapshot names.
169 * Note that snapshots in ascending order of snapshot id are stored in
170 * {@link INodeDirectoryWithSnapshot}.diffs (a private field).
171 */
172 private final List<Snapshot> snapshotsByNames = new ArrayList<Snapshot>();
173
174 /**
175 * @return {@link #snapshotsByNames}
176 */
177 ReadOnlyList<Snapshot> getSnapshotsByNames() {
178 return ReadOnlyList.Util.asReadOnlyList(this.snapshotsByNames);
179 }
180
181 /** Number of snapshots allowed. */
182 private int snapshotQuota = SNAPSHOT_LIMIT;
183
184 public INodeDirectorySnapshottable(INodeDirectory dir) {
185 super(dir, true, dir instanceof INodeDirectoryWithSnapshot ?
186 ((INodeDirectoryWithSnapshot) dir).getDiffs(): null);
187 }
188
189 /** @return the number of existing snapshots. */
190 public int getNumSnapshots() {
191 return snapshotsByNames.size();
192 }
193
194 private int searchSnapshot(byte[] snapshotName) {
195 return Collections.binarySearch(snapshotsByNames, snapshotName);
196 }
197
198 /** @return the snapshot with the given name. */
199 public Snapshot getSnapshot(byte[] snapshotName) {
200 final int i = searchSnapshot(snapshotName);
201 return i < 0? null: snapshotsByNames.get(i);
202 }
203
204 /** @return {@link #snapshotsByNames} as a {@link ReadOnlyList} */
205 public ReadOnlyList<Snapshot> getSnapshotList() {
206 return ReadOnlyList.Util.asReadOnlyList(snapshotsByNames);
207 }
208
209 /**
210 * Rename a snapshot
211 * @param path
212 * The directory path where the snapshot was taken. Used for
213 * generating exception message.
214 * @param oldName
215 * Old name of the snapshot
216 * @param newName
217 * New name the snapshot will be renamed to
218 * @throws SnapshotException
219 * Throw SnapshotException when either the snapshot with the old
220 * name does not exist or a snapshot with the new name already
221 * exists
222 */
223 public void renameSnapshot(String path, String oldName, String newName)
224 throws SnapshotException {
225 if (newName.equals(oldName)) {
226 return;
227 }
228 final int indexOfOld = searchSnapshot(DFSUtil.string2Bytes(oldName));
229 if (indexOfOld < 0) {
230 throw new SnapshotException("The snapshot " + oldName
231 + " does not exist for directory " + path);
232 } else {
233 final byte[] newNameBytes = DFSUtil.string2Bytes(newName);
234 int indexOfNew = searchSnapshot(newNameBytes);
235 if (indexOfNew > 0) {
236 throw new SnapshotException("The snapshot " + newName
237 + " already exists for directory " + path);
238 }
239 // remove the one with old name from snapshotsByNames
240 Snapshot snapshot = snapshotsByNames.remove(indexOfOld);
241 final INodeDirectory ssRoot = snapshot.getRoot();
242 ssRoot.setLocalName(newNameBytes);
243 indexOfNew = -indexOfNew - 1;
244 if (indexOfNew <= indexOfOld) {
245 snapshotsByNames.add(indexOfNew, snapshot);
246 } else { // indexOfNew > indexOfOld
247 snapshotsByNames.add(indexOfNew - 1, snapshot);
248 }
249 }
250 }
251
252 public int getSnapshotQuota() {
253 return snapshotQuota;
254 }
255
256 public void setSnapshotQuota(int snapshotQuota) {
257 if (snapshotQuota < 0) {
258 throw new HadoopIllegalArgumentException(
259 "Cannot set snapshot quota to " + snapshotQuota + " < 0");
260 }
261 this.snapshotQuota = snapshotQuota;
262 }
263
264 @Override
265 public boolean isSnapshottable() {
266 return true;
267 }
268
269 /**
270 * Simply add a snapshot into the {@link #snapshotsByNames}. Used by FSImage
271 * loading.
272 */
273 void addSnapshot(Snapshot snapshot) {
274 this.snapshotsByNames.add(snapshot);
275 }
276
277 /** Add a snapshot. */
278 Snapshot addSnapshot(int id, String name) throws SnapshotException,
279 QuotaExceededException {
280 //check snapshot quota
281 final int n = getNumSnapshots();
282 if (n + 1 > snapshotQuota) {
283 throw new SnapshotException("Failed to add snapshot: there are already "
284 + n + " snapshot(s) and the snapshot quota is "
285 + snapshotQuota);
286 }
287 final Snapshot s = new Snapshot(id, name, this);
288 final byte[] nameBytes = s.getRoot().getLocalNameBytes();
289 final int i = searchSnapshot(nameBytes);
290 if (i >= 0) {
291 throw new SnapshotException("Failed to add snapshot: there is already a "
292 + "snapshot with the same name \"" + Snapshot.getSnapshotName(s) + "\".");
293 }
294
295 final DirectoryDiff d = getDiffs().addDiff(s, this);
296 d.snapshotINode = s.getRoot();
297 snapshotsByNames.add(-i - 1, s);
298
299 //set modification time
300 updateModificationTime(Time.now(), null, null);
301 s.getRoot().setModificationTime(getModificationTime(), null, null);
302 return s;
303 }
304
305 /**
306 * Remove the snapshot with the given name from {@link #snapshotsByNames},
307 * and delete all the corresponding DirectoryDiff.
308 *
309 * @param snapshotName The name of the snapshot to be removed
310 * @param collectedBlocks Used to collect information to update blocksMap
311 * @return The removed snapshot. Null if no snapshot with the given name
312 * exists.
313 */
314 Snapshot removeSnapshot(String snapshotName,
315 BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
316 throws SnapshotException {
317 final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
318 if (i < 0) {
319 throw new SnapshotException("Cannot delete snapshot " + snapshotName
320 + " from path " + this.getFullPathName()
321 + ": the snapshot does not exist.");
322 } else {
323 final Snapshot snapshot = snapshotsByNames.get(i);
324 Snapshot prior = Snapshot.findLatestSnapshot(this, snapshot);
325 try {
326 Quota.Counts counts = cleanSubtree(snapshot, prior, collectedBlocks,
327 removedINodes, true);
328 INodeDirectory parent = getParent();
329 if (parent != null) {
330 // there will not be any WithName node corresponding to the deleted
331 // snapshot, thus only update the quota usage in the current tree
332 parent.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
333 -counts.get(Quota.DISKSPACE), true);
334 }
335 } catch(QuotaExceededException e) {
336 LOG.error("BUG: removeSnapshot increases namespace usage.", e);
337 }
338 // remove from snapshotsByNames after successfully cleaning the subtree
339 snapshotsByNames.remove(i);
340 return snapshot;
341 }
342 }
343
344 @Override
345 public Content.Counts computeContentSummary(final Content.Counts counts) {
346 super.computeContentSummary(counts);
347 counts.add(Content.SNAPSHOT, snapshotsByNames.size());
348 counts.add(Content.SNAPSHOTTABLE_DIRECTORY, 1);
349 return counts;
350 }
351
352 /**
353 * Compute the difference between two snapshots (or a snapshot and the current
354 * directory) of the directory.
355 *
356 * @param from The name of the start point of the comparison. Null indicating
357 * the current tree.
358 * @param to The name of the end point. Null indicating the current tree.
359 * @return The difference between the start/end points.
360 * @throws SnapshotException If there is no snapshot matching the starting
361 * point, or if endSnapshotName is not null but cannot be identified
362 * as a previous snapshot.
363 */
364 SnapshotDiffInfo computeDiff(final String from, final String to)
365 throws SnapshotException {
366 Snapshot fromSnapshot = getSnapshotByName(from);
367 Snapshot toSnapshot = getSnapshotByName(to);
368 // if the start point is equal to the end point, return null
369 if (from.equals(to)) {
370 return null;
371 }
372 SnapshotDiffInfo diffs = new SnapshotDiffInfo(this, fromSnapshot,
373 toSnapshot);
374 computeDiffRecursively(this, new ArrayList<byte[]>(), diffs);
375 return diffs;
376 }
377
378 /**
379 * Find the snapshot matching the given name.
380 *
381 * @param snapshotName The name of the snapshot.
382 * @return The corresponding snapshot. Null if snapshotName is null or empty.
383 * @throws SnapshotException If snapshotName is not null or empty, but there
384 * is no snapshot matching the name.
385 */
386 private Snapshot getSnapshotByName(String snapshotName)
387 throws SnapshotException {
388 Snapshot s = null;
389 if (snapshotName != null && !snapshotName.isEmpty()) {
390 final int index = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
391 if (index < 0) {
392 throw new SnapshotException("Cannot find the snapshot of directory "
393 + this.getFullPathName() + " with name " + snapshotName);
394 }
395 s = snapshotsByNames.get(index);
396 }
397 return s;
398 }
399
400 /**
401 * Recursively compute the difference between snapshots under a given
402 * directory/file.
403 * @param node The directory/file under which the diff is computed.
404 * @param parentPath Relative path (corresponding to the snapshot root) of
405 * the node's parent.
406 * @param diffReport data structure used to store the diff.
407 */
408 private void computeDiffRecursively(INode node, List<byte[]> parentPath,
409 SnapshotDiffInfo diffReport) {
410 ChildrenDiff diff = new ChildrenDiff();
411 byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
412 if (node.isDirectory()) {
413 INodeDirectory dir = node.asDirectory();
414 if (dir instanceof INodeDirectoryWithSnapshot) {
415 INodeDirectoryWithSnapshot sdir = (INodeDirectoryWithSnapshot) dir;
416 boolean change = sdir.computeDiffBetweenSnapshots(
417 diffReport.from, diffReport.to, diff);
418 if (change) {
419 diffReport.addDirDiff(sdir, relativePath, diff);
420 }
421 }
422 ReadOnlyList<INode> children = dir.getChildrenList(diffReport
423 .isFromEarlier() ? diffReport.to : diffReport.from);
424 for (INode child : children) {
425 final byte[] name = child.getLocalNameBytes();
426 if (diff.searchIndex(ListType.CREATED, name) < 0
427 && diff.searchIndex(ListType.DELETED, name) < 0) {
428 parentPath.add(name);
429 computeDiffRecursively(child, parentPath, diffReport);
430 parentPath.remove(parentPath.size() - 1);
431 }
432 }
433 } else if (node.isFile() && node.asFile() instanceof FileWithSnapshot) {
434 FileWithSnapshot file = (FileWithSnapshot) node.asFile();
435 Snapshot earlierSnapshot = diffReport.isFromEarlier() ? diffReport.from
436 : diffReport.to;
437 Snapshot laterSnapshot = diffReport.isFromEarlier() ? diffReport.to
438 : diffReport.from;
439 boolean change = file.getDiffs().changedBetweenSnapshots(earlierSnapshot,
440 laterSnapshot);
441 if (change) {
442 diffReport.addFileDiff(file.asINodeFile(), relativePath);
443 }
444 }
445 }
446
447 /**
448 * Replace itself with {@link INodeDirectoryWithSnapshot} or
449 * {@link INodeDirectory} depending on the latest snapshot.
450 */
451 INodeDirectory replaceSelf(final Snapshot latest, final INodeMap inodeMap)
452 throws QuotaExceededException {
453 if (latest == null) {
454 Preconditions.checkState(getLastSnapshot() == null,
455 "latest == null but getLastSnapshot() != null, this=%s", this);
456 return replaceSelf4INodeDirectory(inodeMap);
457 } else {
458 return replaceSelf4INodeDirectoryWithSnapshot(inodeMap)
459 .recordModification(latest, null);
460 }
461 }
462
463 @Override
464 public String toDetailString() {
465 return super.toDetailString() + ", snapshotsByNames=" + snapshotsByNames;
466 }
467
468 @Override
469 public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
470 Snapshot snapshot) {
471 super.dumpTreeRecursively(out, prefix, snapshot);
472
473 if (snapshot == null) {
474 out.println();
475 out.print(prefix);
476
477 out.print("Snapshot of ");
478 final String name = getLocalName();
479 out.print(name.isEmpty()? "/": name);
480 out.print(": quota=");
481 out.print(getSnapshotQuota());
482
483 int n = 0;
484 for(DirectoryDiff diff : getDiffs()) {
485 if (diff.isSnapshotRoot()) {
486 n++;
487 }
488 }
489 Preconditions.checkState(n == snapshotsByNames.size());
490 out.print(", #snapshot=");
491 out.println(n);
492
493 dumpTreeRecursively(out, prefix, new Iterable<SnapshotAndINode>() {
494 @Override
495 public Iterator<SnapshotAndINode> iterator() {
496 return new Iterator<SnapshotAndINode>() {
497 final Iterator<DirectoryDiff> i = getDiffs().iterator();
498 private DirectoryDiff next = findNext();
499
500 private DirectoryDiff findNext() {
501 for(; i.hasNext(); ) {
502 final DirectoryDiff diff = i.next();
503 if (diff.isSnapshotRoot()) {
504 return diff;
505 }
506 }
507 return null;
508 }
509
510 @Override
511 public boolean hasNext() {
512 return next != null;
513 }
514
515 @Override
516 public SnapshotAndINode next() {
517 final Snapshot s = next.snapshot;
518 final SnapshotAndINode pair = new SnapshotAndINode(s);
519 next = findNext();
520 return pair;
521 }
522
523 @Override
524 public void remove() {
525 throw new UnsupportedOperationException();
526 }
527 };
528 }
529 });
530 }
531 }
532 }