001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.namenode.snapshot;
019
020 import java.io.DataInput;
021 import java.io.DataOutput;
022 import java.io.IOException;
023 import java.util.ArrayList;
024 import java.util.HashMap;
025 import java.util.List;
026 import java.util.Map;
027
028 import org.apache.hadoop.hdfs.DFSUtil;
029 import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
030 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
031 import org.apache.hadoop.hdfs.server.namenode.INode;
032 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
033 import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
034 import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
035 import org.apache.hadoop.hdfs.server.namenode.INodeReference;
036 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
037 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
038 import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
039 import org.apache.hadoop.hdfs.util.Diff.ListType;
040 import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader;
041
042 /**
043 * A helper class defining static methods for reading/writing snapshot related
044 * information from/to FSImage.
045 */
046 public class SnapshotFSImageFormat {
047 public static FileDiffList loadFileDiffList(DataInput in,
048 FSImageFormat.Loader loader) throws IOException {
049 final int size = in.readInt();
050 if (size == -1) {
051 return null;
052 } else {
053 final FileDiffList diffs = new FileDiffList();
054 FileDiff posterior = null;
055 for(int i = 0; i < size; i++) {
056 final FileDiff d = loadFileDiff(posterior, in, loader);
057 diffs.addFirst(d);
058 posterior = d;
059 }
060 return diffs;
061 }
062 }
063
064 private static FileDiff loadFileDiff(FileDiff posterior, DataInput in,
065 FSImageFormat.Loader loader) throws IOException {
066 // 1. Read the id of the Snapshot root to identify the Snapshot
067 final Snapshot snapshot = loader.getSnapshot(in);
068
069 // 2. Load file size
070 final long fileSize = in.readLong();
071
072 // 3. Load snapshotINode
073 final INodeFileAttributes snapshotINode = in.readBoolean()?
074 loader.loadINodeFileAttributes(in): null;
075
076 return new FileDiff(snapshot.getId(), snapshotINode, posterior, fileSize);
077 }
078
079 /**
080 * Load a node stored in the created list from fsimage.
081 * @param createdNodeName The name of the created node.
082 * @param parent The directory that the created list belongs to.
083 * @return The created node.
084 */
085 public static INode loadCreated(byte[] createdNodeName,
086 INodeDirectory parent) throws IOException {
087 // the INode in the created list should be a reference to another INode
088 // in posterior SnapshotDiffs or one of the current children
089 for (DirectoryDiff postDiff : parent.getDiffs()) {
090 final INode d = postDiff.getChildrenDiff().search(ListType.DELETED,
091 createdNodeName);
092 if (d != null) {
093 return d;
094 } // else go to the next SnapshotDiff
095 }
096 // use the current child
097 INode currentChild = parent.getChild(createdNodeName,
098 Snapshot.CURRENT_STATE_ID);
099 if (currentChild == null) {
100 throw new IOException("Cannot find an INode associated with the INode "
101 + DFSUtil.bytes2String(createdNodeName)
102 + " in created list while loading FSImage.");
103 }
104 return currentChild;
105 }
106
107 /**
108 * Load the created list from fsimage.
109 * @param parent The directory that the created list belongs to.
110 * @param in The {@link DataInput} to read.
111 * @return The created list.
112 */
113 private static List<INode> loadCreatedList(INodeDirectory parent,
114 DataInput in) throws IOException {
115 // read the size of the created list
116 int createdSize = in.readInt();
117 List<INode> createdList = new ArrayList<INode>(createdSize);
118 for (int i = 0; i < createdSize; i++) {
119 byte[] createdNodeName = FSImageSerialization.readLocalName(in);
120 INode created = loadCreated(createdNodeName, parent);
121 createdList.add(created);
122 }
123 return createdList;
124 }
125
126 /**
127 * Load the deleted list from the fsimage.
128 *
129 * @param parent The directory that the deleted list belongs to.
130 * @param createdList The created list associated with the deleted list in
131 * the same Diff.
132 * @param in The {@link DataInput} to read.
133 * @param loader The {@link Loader} instance.
134 * @return The deleted list.
135 */
136 private static List<INode> loadDeletedList(INodeDirectory parent,
137 List<INode> createdList, DataInput in, FSImageFormat.Loader loader)
138 throws IOException {
139 int deletedSize = in.readInt();
140 List<INode> deletedList = new ArrayList<INode>(deletedSize);
141 for (int i = 0; i < deletedSize; i++) {
142 final INode deleted = loader.loadINodeWithLocalName(true, in, true);
143 deletedList.add(deleted);
144 // set parent: the parent field of an INode in the deleted list is not
145 // useful, but set the parent here to be consistent with the original
146 // fsdir tree.
147 deleted.setParent(parent);
148 if (deleted.isFile()) {
149 loader.updateBlocksMap(deleted.asFile());
150 }
151 }
152 return deletedList;
153 }
154
155 /**
156 * Load snapshots and snapshotQuota for a Snapshottable directory.
157 *
158 * @param snapshottableParent
159 * The snapshottable directory for loading.
160 * @param numSnapshots
161 * The number of snapshots that the directory has.
162 * @param loader
163 * The loader
164 */
165 public static void loadSnapshotList(
166 INodeDirectorySnapshottable snapshottableParent, int numSnapshots,
167 DataInput in, FSImageFormat.Loader loader) throws IOException {
168 for (int i = 0; i < numSnapshots; i++) {
169 // read snapshots
170 final Snapshot s = loader.getSnapshot(in);
171 s.getRoot().setParent(snapshottableParent);
172 snapshottableParent.addSnapshot(s);
173 }
174 int snapshotQuota = in.readInt();
175 snapshottableParent.setSnapshotQuota(snapshotQuota);
176 }
177
178 /**
179 * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
180 * directory.
181 *
182 * @param dir
183 * The snapshottable directory for loading.
184 * @param in
185 * The {@link DataInput} instance to read.
186 * @param loader
187 * The loader
188 */
189 public static void loadDirectoryDiffList(INodeDirectory dir,
190 DataInput in, FSImageFormat.Loader loader) throws IOException {
191 final int size = in.readInt();
192 if (dir.isWithSnapshot()) {
193 DirectoryDiffList diffs = dir.getDiffs();
194 for (int i = 0; i < size; i++) {
195 diffs.addFirst(loadDirectoryDiff(dir, in, loader));
196 }
197 }
198 }
199
200 /**
201 * Load the snapshotINode field of {@link AbstractINodeDiff}.
202 * @param snapshot The Snapshot associated with the {@link AbstractINodeDiff}.
203 * @param in The {@link DataInput} to read.
204 * @param loader The {@link Loader} instance that this loading procedure is
205 * using.
206 * @return The snapshotINode.
207 */
208 private static INodeDirectoryAttributes loadSnapshotINodeInDirectoryDiff(
209 Snapshot snapshot, DataInput in, FSImageFormat.Loader loader)
210 throws IOException {
211 // read the boolean indicating whether snapshotINode == Snapshot.Root
212 boolean useRoot = in.readBoolean();
213 if (useRoot) {
214 return snapshot.getRoot();
215 } else {
216 // another boolean is used to indicate whether snapshotINode is non-null
217 return in.readBoolean()? loader.loadINodeDirectoryAttributes(in): null;
218 }
219 }
220
221 /**
222 * Load {@link DirectoryDiff} from fsimage.
223 * @param parent The directory that the SnapshotDiff belongs to.
224 * @param in The {@link DataInput} instance to read.
225 * @param loader The {@link Loader} instance that this loading procedure is
226 * using.
227 * @return A {@link DirectoryDiff}.
228 */
229 private static DirectoryDiff loadDirectoryDiff(INodeDirectory parent,
230 DataInput in, FSImageFormat.Loader loader) throws IOException {
231 // 1. Read the full path of the Snapshot root to identify the Snapshot
232 final Snapshot snapshot = loader.getSnapshot(in);
233
234 // 2. Load DirectoryDiff#childrenSize
235 int childrenSize = in.readInt();
236
237 // 3. Load DirectoryDiff#snapshotINode
238 INodeDirectoryAttributes snapshotINode = loadSnapshotINodeInDirectoryDiff(
239 snapshot, in, loader);
240
241 // 4. Load the created list in SnapshotDiff#Diff
242 List<INode> createdList = loadCreatedList(parent, in);
243
244 // 5. Load the deleted list in SnapshotDiff#Diff
245 List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);
246
247 // 6. Compose the SnapshotDiff
248 List<DirectoryDiff> diffs = parent.getDiffs().asList();
249 DirectoryDiff sdiff = new DirectoryDiff(snapshot.getId(), snapshotINode,
250 diffs.isEmpty() ? null : diffs.get(0), childrenSize, createdList,
251 deletedList, snapshotINode == snapshot.getRoot());
252 return sdiff;
253 }
254
255
256 /** A reference map for fsimage serialization. */
257 public static class ReferenceMap {
258 /**
259 * Used to indicate whether the reference node itself has been saved
260 */
261 private final Map<Long, INodeReference.WithCount> referenceMap
262 = new HashMap<Long, INodeReference.WithCount>();
263 /**
264 * Used to record whether the subtree of the reference node has been saved
265 */
266 private final Map<Long, Long> dirMap = new HashMap<Long, Long>();
267
268 public boolean toProcessSubtree(long id) {
269 if (dirMap.containsKey(id)) {
270 return false;
271 } else {
272 dirMap.put(id, id);
273 return true;
274 }
275 }
276
277 public INodeReference.WithCount loadINodeReferenceWithCount(
278 boolean isSnapshotINode, DataInput in, FSImageFormat.Loader loader
279 ) throws IOException {
280 final boolean firstReferred = in.readBoolean();
281
282 final INodeReference.WithCount withCount;
283 if (firstReferred) {
284 final INode referred = loader.loadINodeWithLocalName(isSnapshotINode,
285 in, true);
286 withCount = new INodeReference.WithCount(null, referred);
287 referenceMap.put(withCount.getId(), withCount);
288 } else {
289 final long id = in.readLong();
290 withCount = referenceMap.get(id);
291 }
292 return withCount;
293 }
294 }
295 }