001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.namenode.snapshot;
019
020 import java.io.DataInput;
021 import java.io.DataOutput;
022 import java.io.IOException;
023 import java.util.ArrayList;
024 import java.util.HashMap;
025 import java.util.List;
026 import java.util.Map;
027
028 import org.apache.hadoop.hdfs.DFSUtil;
029 import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
030 import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader;
031 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
032 import org.apache.hadoop.hdfs.server.namenode.INode;
033 import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
034 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
035 import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
036 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
037 import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
038 import org.apache.hadoop.hdfs.server.namenode.INodeReference;
039 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot.FileDiff;
040 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot.FileDiffList;
041 import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.DirectoryDiff;
042 import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.DirectoryDiffList;
043 import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
044 import org.apache.hadoop.hdfs.util.Diff.ListType;
045 import org.apache.hadoop.hdfs.util.ReadOnlyList;
046
047 /**
048 * A helper class defining static methods for reading/writing snapshot related
049 * information from/to FSImage.
050 */
051 public class SnapshotFSImageFormat {
052 /**
053 * Save snapshots and snapshot quota for a snapshottable directory.
054 * @param current The directory that the snapshots belongs to.
055 * @param out The {@link DataOutput} to write.
056 * @throws IOException
057 */
058 public static void saveSnapshots(INodeDirectorySnapshottable current,
059 DataOutput out) throws IOException {
060 // list of snapshots in snapshotsByNames
061 ReadOnlyList<Snapshot> snapshots = current.getSnapshotsByNames();
062 out.writeInt(snapshots.size());
063 for (Snapshot s : snapshots) {
064 // write the snapshot id
065 out.writeInt(s.getId());
066 }
067 // snapshot quota
068 out.writeInt(current.getSnapshotQuota());
069 }
070
071 /**
072 * Save SnapshotDiff list for an INodeDirectoryWithSnapshot.
073 * @param sNode The directory that the SnapshotDiff list belongs to.
074 * @param out The {@link DataOutput} to write.
075 */
076 private static <N extends INode, A extends INodeAttributes, D extends AbstractINodeDiff<N, A, D>>
077 void saveINodeDiffs(final AbstractINodeDiffList<N, A, D> diffs,
078 final DataOutput out, ReferenceMap referenceMap) throws IOException {
079 // Record the diffs in reversed order, so that we can find the correct
080 // reference for INodes in the created list when loading the FSImage
081 if (diffs == null) {
082 out.writeInt(-1); // no diffs
083 } else {
084 final List<D> list = diffs.asList();
085 final int size = list.size();
086 out.writeInt(size);
087 for (int i = size - 1; i >= 0; i--) {
088 list.get(i).write(out, referenceMap);
089 }
090 }
091 }
092
093 public static void saveDirectoryDiffList(final INodeDirectory dir,
094 final DataOutput out, final ReferenceMap referenceMap
095 ) throws IOException {
096 saveINodeDiffs(dir instanceof INodeDirectoryWithSnapshot?
097 ((INodeDirectoryWithSnapshot)dir).getDiffs(): null, out, referenceMap);
098 }
099
100 public static void saveFileDiffList(final INodeFile file,
101 final DataOutput out) throws IOException {
102 saveINodeDiffs(file instanceof FileWithSnapshot?
103 ((FileWithSnapshot)file).getDiffs(): null, out, null);
104 }
105
106 public static FileDiffList loadFileDiffList(DataInput in,
107 FSImageFormat.Loader loader) throws IOException {
108 final int size = in.readInt();
109 if (size == -1) {
110 return null;
111 } else {
112 final FileDiffList diffs = new FileDiffList();
113 FileDiff posterior = null;
114 for(int i = 0; i < size; i++) {
115 final FileDiff d = loadFileDiff(posterior, in, loader);
116 diffs.addFirst(d);
117 posterior = d;
118 }
119 return diffs;
120 }
121 }
122
123 private static FileDiff loadFileDiff(FileDiff posterior, DataInput in,
124 FSImageFormat.Loader loader) throws IOException {
125 // 1. Read the full path of the Snapshot root to identify the Snapshot
126 final Snapshot snapshot = loader.getSnapshot(in);
127
128 // 2. Load file size
129 final long fileSize = in.readLong();
130
131 // 3. Load snapshotINode
132 final INodeFileAttributes snapshotINode = in.readBoolean()?
133 loader.loadINodeFileAttributes(in): null;
134
135 return new FileDiff(snapshot, snapshotINode, posterior, fileSize);
136 }
137
138 /**
139 * Load a node stored in the created list from fsimage.
140 * @param createdNodeName The name of the created node.
141 * @param parent The directory that the created list belongs to.
142 * @return The created node.
143 */
144 private static INode loadCreated(byte[] createdNodeName,
145 INodeDirectoryWithSnapshot parent) throws IOException {
146 // the INode in the created list should be a reference to another INode
147 // in posterior SnapshotDiffs or one of the current children
148 for (DirectoryDiff postDiff : parent.getDiffs()) {
149 final INode d = postDiff.getChildrenDiff().search(ListType.DELETED,
150 createdNodeName);
151 if (d != null) {
152 return d;
153 } // else go to the next SnapshotDiff
154 }
155 // use the current child
156 INode currentChild = parent.getChild(createdNodeName, null);
157 if (currentChild == null) {
158 throw new IOException("Cannot find an INode associated with the INode "
159 + DFSUtil.bytes2String(createdNodeName)
160 + " in created list while loading FSImage.");
161 }
162 return currentChild;
163 }
164
165 /**
166 * Load the created list from fsimage.
167 * @param parent The directory that the created list belongs to.
168 * @param in The {@link DataInput} to read.
169 * @return The created list.
170 */
171 private static List<INode> loadCreatedList(INodeDirectoryWithSnapshot parent,
172 DataInput in) throws IOException {
173 // read the size of the created list
174 int createdSize = in.readInt();
175 List<INode> createdList = new ArrayList<INode>(createdSize);
176 for (int i = 0; i < createdSize; i++) {
177 byte[] createdNodeName = FSImageSerialization.readLocalName(in);
178 INode created = loadCreated(createdNodeName, parent);
179 createdList.add(created);
180 }
181 return createdList;
182 }
183
184 /**
185 * Load the deleted list from the fsimage.
186 *
187 * @param parent The directory that the deleted list belongs to.
188 * @param createdList The created list associated with the deleted list in
189 * the same Diff.
190 * @param in The {@link DataInput} to read.
191 * @param loader The {@link Loader} instance.
192 * @return The deleted list.
193 */
194 private static List<INode> loadDeletedList(INodeDirectoryWithSnapshot parent,
195 List<INode> createdList, DataInput in, FSImageFormat.Loader loader)
196 throws IOException {
197 int deletedSize = in.readInt();
198 List<INode> deletedList = new ArrayList<INode>(deletedSize);
199 for (int i = 0; i < deletedSize; i++) {
200 final INode deleted = loader.loadINodeWithLocalName(true, in, true);
201 deletedList.add(deleted);
202 // set parent: the parent field of an INode in the deleted list is not
203 // useful, but set the parent here to be consistent with the original
204 // fsdir tree.
205 deleted.setParent(parent);
206 }
207 return deletedList;
208 }
209
210 /**
211 * Load snapshots and snapshotQuota for a Snapshottable directory.
212 * @param snapshottableParent The snapshottable directory for loading.
213 * @param numSnapshots The number of snapshots that the directory has.
214 * @param in The {@link DataInput} instance to read.
215 * @param loader The {@link Loader} instance that this loading procedure is
216 * using.
217 */
218 public static void loadSnapshotList(
219 INodeDirectorySnapshottable snapshottableParent, int numSnapshots,
220 DataInput in, FSImageFormat.Loader loader) throws IOException {
221 for (int i = 0; i < numSnapshots; i++) {
222 // read snapshots
223 final Snapshot s = loader.getSnapshot(in);
224 s.getRoot().setParent(snapshottableParent);
225 snapshottableParent.addSnapshot(s);
226 }
227 int snapshotQuota = in.readInt();
228 snapshottableParent.setSnapshotQuota(snapshotQuota);
229 }
230
231 /**
232 * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
233 * directory.
234 * @param dir The snapshottable directory for loading.
235 * @param in The {@link DataInput} instance to read.
236 * @param loader The {@link Loader} instance that this loading procedure is
237 * using.
238 */
239 public static void loadDirectoryDiffList(INodeDirectory dir,
240 DataInput in, FSImageFormat.Loader loader) throws IOException {
241 final int size = in.readInt();
242 if (dir instanceof INodeDirectoryWithSnapshot) {
243 INodeDirectoryWithSnapshot withSnapshot = (INodeDirectoryWithSnapshot)dir;
244 DirectoryDiffList diffs = withSnapshot.getDiffs();
245 for (int i = 0; i < size; i++) {
246 diffs.addFirst(loadDirectoryDiff(withSnapshot, in, loader));
247 }
248 }
249 }
250
251 /**
252 * Load the snapshotINode field of {@link AbstractINodeDiff}.
253 * @param snapshot The Snapshot associated with the {@link AbstractINodeDiff}.
254 * @param in The {@link DataInput} to read.
255 * @param loader The {@link Loader} instance that this loading procedure is
256 * using.
257 * @return The snapshotINode.
258 */
259 private static INodeDirectoryAttributes loadSnapshotINodeInDirectoryDiff(
260 Snapshot snapshot, DataInput in, FSImageFormat.Loader loader)
261 throws IOException {
262 // read the boolean indicating whether snapshotINode == Snapshot.Root
263 boolean useRoot = in.readBoolean();
264 if (useRoot) {
265 return snapshot.getRoot();
266 } else {
267 // another boolean is used to indicate whether snapshotINode is non-null
268 return in.readBoolean()? loader.loadINodeDirectoryAttributes(in): null;
269 }
270 }
271
272 /**
273 * Load {@link DirectoryDiff} from fsimage.
274 * @param parent The directory that the SnapshotDiff belongs to.
275 * @param in The {@link DataInput} instance to read.
276 * @param loader The {@link Loader} instance that this loading procedure is
277 * using.
278 * @return A {@link DirectoryDiff}.
279 */
280 private static DirectoryDiff loadDirectoryDiff(
281 INodeDirectoryWithSnapshot parent, DataInput in,
282 FSImageFormat.Loader loader) throws IOException {
283 // 1. Read the full path of the Snapshot root to identify the Snapshot
284 final Snapshot snapshot = loader.getSnapshot(in);
285
286 // 2. Load DirectoryDiff#childrenSize
287 int childrenSize = in.readInt();
288
289 // 3. Load DirectoryDiff#snapshotINode
290 INodeDirectoryAttributes snapshotINode = loadSnapshotINodeInDirectoryDiff(
291 snapshot, in, loader);
292
293 // 4. Load the created list in SnapshotDiff#Diff
294 List<INode> createdList = loadCreatedList(parent, in);
295
296 // 5. Load the deleted list in SnapshotDiff#Diff
297 List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);
298
299 // 6. Compose the SnapshotDiff
300 List<DirectoryDiff> diffs = parent.getDiffs().asList();
301 DirectoryDiff sdiff = new DirectoryDiff(snapshot, snapshotINode,
302 diffs.isEmpty() ? null : diffs.get(0),
303 childrenSize, createdList, deletedList);
304 return sdiff;
305 }
306
307
308 /** A reference map for fsimage serialization. */
309 public static class ReferenceMap {
310 /**
311 * Used to indicate whether the reference node itself has been saved
312 */
313 private final Map<Long, INodeReference.WithCount> referenceMap
314 = new HashMap<Long, INodeReference.WithCount>();
315 /**
316 * Used to record whether the subtree of the reference node has been saved
317 */
318 private final Map<Long, Long> dirMap = new HashMap<Long, Long>();
319
320 public void writeINodeReferenceWithCount(
321 INodeReference.WithCount withCount, DataOutput out,
322 boolean writeUnderConstruction) throws IOException {
323 final INode referred = withCount.getReferredINode();
324 final long id = withCount.getId();
325 final boolean firstReferred = !referenceMap.containsKey(id);
326 out.writeBoolean(firstReferred);
327
328 if (firstReferred) {
329 FSImageSerialization.saveINode2Image(referred, out,
330 writeUnderConstruction, this);
331 referenceMap.put(id, withCount);
332 } else {
333 out.writeLong(id);
334 }
335 }
336
337 public boolean toProcessSubtree(long id) {
338 if (dirMap.containsKey(id)) {
339 return false;
340 } else {
341 dirMap.put(id, id);
342 return true;
343 }
344 }
345
346 public INodeReference.WithCount loadINodeReferenceWithCount(
347 boolean isSnapshotINode, DataInput in, FSImageFormat.Loader loader
348 ) throws IOException {
349 final boolean firstReferred = in.readBoolean();
350
351 final INodeReference.WithCount withCount;
352 if (firstReferred) {
353 final INode referred = loader.loadINodeWithLocalName(isSnapshotINode,
354 in, true);
355 withCount = new INodeReference.WithCount(null, referred);
356 referenceMap.put(withCount.getId(), withCount);
357 } else {
358 final long id = in.readLong();
359 withCount = referenceMap.get(id);
360 }
361 return withCount;
362 }
363 }
364 }