001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.server.namenode;
020
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.io.OutputStream;
024 import java.util.ArrayList;
025 import java.util.Iterator;
026 import java.util.List;
027 import java.util.Map;
028
029 import org.apache.commons.logging.Log;
030 import org.apache.commons.logging.LogFactory;
031 import org.apache.hadoop.HadoopIllegalArgumentException;
032 import org.apache.hadoop.classification.InterfaceAudience;
033 import org.apache.hadoop.fs.permission.AclEntry;
034 import org.apache.hadoop.fs.permission.AclEntryScope;
035 import org.apache.hadoop.fs.permission.AclEntryType;
036 import org.apache.hadoop.fs.permission.FsAction;
037 import org.apache.hadoop.fs.permission.FsPermission;
038 import org.apache.hadoop.fs.permission.PermissionStatus;
039 import org.apache.hadoop.hdfs.protocol.Block;
040 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
041 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
042 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
043 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
044 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
045 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
046 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SaverContext;
047 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
048 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
049 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection;
050 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
051 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.AclFeatureProto;
052 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
053 import org.apache.hadoop.hdfs.util.ReadOnlyList;
054
055 import com.google.common.base.Preconditions;
056 import com.google.common.collect.ImmutableList;
057 import com.google.protobuf.ByteString;
058
059 @InterfaceAudience.Private
060 public final class FSImageFormatPBINode {
061 private final static long USER_GROUP_STRID_MASK = (1 << 24) - 1;
062 private final static int USER_STRID_OFFSET = 40;
063 private final static int GROUP_STRID_OFFSET = 16;
064 private static final Log LOG = LogFactory.getLog(FSImageFormatPBINode.class);
065
066 private static final int ACL_ENTRY_NAME_MASK = (1 << 24) - 1;
067 private static final int ACL_ENTRY_NAME_OFFSET = 6;
068 private static final int ACL_ENTRY_TYPE_OFFSET = 3;
069 private static final int ACL_ENTRY_SCOPE_OFFSET = 5;
070 private static final int ACL_ENTRY_PERM_MASK = 7;
071 private static final int ACL_ENTRY_TYPE_MASK = 3;
072 private static final int ACL_ENTRY_SCOPE_MASK = 1;
073 private static final FsAction[] FSACTION_VALUES = FsAction.values();
074 private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES = AclEntryScope
075 .values();
076 private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType
077 .values();
078
079 public final static class Loader {
080 public static PermissionStatus loadPermission(long id,
081 final String[] stringTable) {
082 short perm = (short) (id & ((1 << GROUP_STRID_OFFSET) - 1));
083 int gsid = (int) ((id >> GROUP_STRID_OFFSET) & USER_GROUP_STRID_MASK);
084 int usid = (int) ((id >> USER_STRID_OFFSET) & USER_GROUP_STRID_MASK);
085 return new PermissionStatus(stringTable[usid], stringTable[gsid],
086 new FsPermission(perm));
087 }
088
089 public static ImmutableList<AclEntry> loadAclEntries(
090 AclFeatureProto proto, final String[] stringTable) {
091 ImmutableList.Builder<AclEntry> b = ImmutableList.builder();
092 for (int v : proto.getEntriesList()) {
093 int p = v & ACL_ENTRY_PERM_MASK;
094 int t = (v >> ACL_ENTRY_TYPE_OFFSET) & ACL_ENTRY_TYPE_MASK;
095 int s = (v >> ACL_ENTRY_SCOPE_OFFSET) & ACL_ENTRY_SCOPE_MASK;
096 int nid = (v >> ACL_ENTRY_NAME_OFFSET) & ACL_ENTRY_NAME_MASK;
097 String name = stringTable[nid];
098 b.add(new AclEntry.Builder().setName(name)
099 .setPermission(FSACTION_VALUES[p])
100 .setScope(ACL_ENTRY_SCOPE_VALUES[s])
101 .setType(ACL_ENTRY_TYPE_VALUES[t]).build());
102 }
103 return b.build();
104 }
105
106 public static INodeDirectory loadINodeDirectory(INodeSection.INode n,
107 LoaderContext state) {
108 assert n.getType() == INodeSection.INode.Type.DIRECTORY;
109 INodeSection.INodeDirectory d = n.getDirectory();
110
111 final PermissionStatus permissions = loadPermission(d.getPermission(),
112 state.getStringTable());
113 final INodeDirectory dir = new INodeDirectory(n.getId(), n.getName()
114 .toByteArray(), permissions, d.getModificationTime());
115
116 final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota();
117 if (nsQuota >= 0 || dsQuota >= 0) {
118 dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota);
119 }
120
121 if (d.hasAcl()) {
122 dir.addAclFeature(new AclFeature(loadAclEntries(d.getAcl(),
123 state.getStringTable())));
124 }
125 return dir;
126 }
127
128 public static void updateBlocksMap(INodeFile file, BlockManager bm) {
129 // Add file->block mapping
130 final BlockInfo[] blocks = file.getBlocks();
131 if (blocks != null) {
132 for (int i = 0; i < blocks.length; i++) {
133 file.setBlock(i, bm.addBlockCollection(blocks[i], file));
134 }
135 }
136 }
137
138 private final FSDirectory dir;
139 private final FSNamesystem fsn;
140 private final FSImageFormatProtobuf.Loader parent;
141
142 Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) {
143 this.fsn = fsn;
144 this.dir = fsn.dir;
145 this.parent = parent;
146 }
147
148 void loadINodeDirectorySection(InputStream in) throws IOException {
149 final List<INodeReference> refList = parent.getLoaderContext()
150 .getRefList();
151 while (true) {
152 INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry
153 .parseDelimitedFrom(in);
154 // note that in is a LimitedInputStream
155 if (e == null) {
156 break;
157 }
158 INodeDirectory p = dir.getInode(e.getParent()).asDirectory();
159 for (long id : e.getChildrenList()) {
160 INode child = dir.getInode(id);
161 addToParent(p, child);
162 }
163 for (int refId : e.getRefChildrenList()) {
164 INodeReference ref = refList.get(refId);
165 addToParent(p, ref);
166 }
167 }
168 }
169
170 void loadINodeSection(InputStream in) throws IOException {
171 INodeSection s = INodeSection.parseDelimitedFrom(in);
172 fsn.resetLastInodeId(s.getLastInodeId());
173 LOG.info("Loading " + s.getNumInodes() + " INodes.");
174 for (int i = 0; i < s.getNumInodes(); ++i) {
175 INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in);
176 if (p.getId() == INodeId.ROOT_INODE_ID) {
177 loadRootINode(p);
178 } else {
179 INode n = loadINode(p);
180 dir.addToInodeMap(n);
181 }
182 }
183 }
184
185 /**
186 * Load the under-construction files section, and update the lease map
187 */
188 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
189 while (true) {
190 FileUnderConstructionEntry entry = FileUnderConstructionEntry
191 .parseDelimitedFrom(in);
192 if (entry == null) {
193 break;
194 }
195 // update the lease manager
196 INodeFile file = dir.getInode(entry.getInodeId()).asFile();
197 FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
198 Preconditions.checkState(uc != null); // file must be under-construction
199 fsn.leaseManager.addLease(uc.getClientName(), entry.getFullPath());
200 }
201 }
202
203 private void addToParent(INodeDirectory parent, INode child) {
204 if (parent == dir.rootDir && FSDirectory.isReservedName(child)) {
205 throw new HadoopIllegalArgumentException("File name \""
206 + child.getLocalName() + "\" is reserved. Please "
207 + " change the name of the existing file or directory to another "
208 + "name before upgrading to this release.");
209 }
210 // NOTE: This does not update space counts for parents
211 if (!parent.addChild(child)) {
212 return;
213 }
214 dir.cacheName(child);
215
216 if (child.isFile()) {
217 updateBlocksMap(child.asFile(), fsn.getBlockManager());
218 }
219 }
220
221 private INode loadINode(INodeSection.INode n) {
222 switch (n.getType()) {
223 case FILE:
224 return loadINodeFile(n);
225 case DIRECTORY:
226 return loadINodeDirectory(n, parent.getLoaderContext());
227 case SYMLINK:
228 return loadINodeSymlink(n);
229 default:
230 break;
231 }
232 return null;
233 }
234
235 private INodeFile loadINodeFile(INodeSection.INode n) {
236 assert n.getType() == INodeSection.INode.Type.FILE;
237 INodeSection.INodeFile f = n.getFile();
238 List<BlockProto> bp = f.getBlocksList();
239 short replication = (short) f.getReplication();
240 LoaderContext state = parent.getLoaderContext();
241
242 BlockInfo[] blocks = new BlockInfo[bp.size()];
243 for (int i = 0, e = bp.size(); i < e; ++i) {
244 blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
245 }
246 final PermissionStatus permissions = loadPermission(f.getPermission(),
247 parent.getLoaderContext().getStringTable());
248
249 final INodeFile file = new INodeFile(n.getId(),
250 n.getName().toByteArray(), permissions, f.getModificationTime(),
251 f.getAccessTime(), blocks, replication, f.getPreferredBlockSize());
252
253 if (f.hasAcl()) {
254 file.addAclFeature(new AclFeature(loadAclEntries(f.getAcl(),
255 state.getStringTable())));
256 }
257
258 // under-construction information
259 if (f.hasFileUC()) {
260 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
261 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine(),
262 null);
263 if (blocks.length > 0) {
264 BlockInfo lastBlk = file.getLastBlock();
265 // replace the last block of file
266 file.setBlock(file.numBlocks() - 1, new BlockInfoUnderConstruction(
267 lastBlk, replication));
268 }
269 }
270 return file;
271 }
272
273
274 private INodeSymlink loadINodeSymlink(INodeSection.INode n) {
275 assert n.getType() == INodeSection.INode.Type.SYMLINK;
276 INodeSection.INodeSymlink s = n.getSymlink();
277 final PermissionStatus permissions = loadPermission(s.getPermission(),
278 parent.getLoaderContext().getStringTable());
279 INodeSymlink sym = new INodeSymlink(n.getId(), n.getName().toByteArray(),
280 permissions, s.getModificationTime(), s.getAccessTime(),
281 s.getTarget().toStringUtf8());
282 return sym;
283 }
284
285 private void loadRootINode(INodeSection.INode p) {
286 INodeDirectory root = loadINodeDirectory(p, parent.getLoaderContext());
287 final Quota.Counts q = root.getQuotaCounts();
288 final long nsQuota = q.get(Quota.NAMESPACE);
289 final long dsQuota = q.get(Quota.DISKSPACE);
290 if (nsQuota != -1 || dsQuota != -1) {
291 dir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
292 }
293 dir.rootDir.cloneModificationTime(root);
294 dir.rootDir.clonePermissionStatus(root);
295 }
296 }
297
298 public final static class Saver {
299 private static long buildPermissionStatus(INodeAttributes n,
300 final SaverContext.DeduplicationMap<String> stringMap) {
301 long userId = stringMap.getId(n.getUserName());
302 long groupId = stringMap.getId(n.getGroupName());
303 return ((userId & USER_GROUP_STRID_MASK) << USER_STRID_OFFSET)
304 | ((groupId & USER_GROUP_STRID_MASK) << GROUP_STRID_OFFSET)
305 | n.getFsPermissionShort();
306 }
307
308 private static AclFeatureProto.Builder buildAclEntries(AclFeature f,
309 final SaverContext.DeduplicationMap<String> map) {
310 AclFeatureProto.Builder b = AclFeatureProto.newBuilder();
311 for (AclEntry e : f.getEntries()) {
312 int v = ((map.getId(e.getName()) & ACL_ENTRY_NAME_MASK) << ACL_ENTRY_NAME_OFFSET)
313 | (e.getType().ordinal() << ACL_ENTRY_TYPE_OFFSET)
314 | (e.getScope().ordinal() << ACL_ENTRY_SCOPE_OFFSET)
315 | (e.getPermission().ordinal());
316 b.addEntries(v);
317 }
318 return b;
319 }
320
321 public static INodeSection.INodeFile.Builder buildINodeFile(
322 INodeFileAttributes file, final SaverContext state) {
323 INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder()
324 .setAccessTime(file.getAccessTime())
325 .setModificationTime(file.getModificationTime())
326 .setPermission(buildPermissionStatus(file, state.getStringMap()))
327 .setPreferredBlockSize(file.getPreferredBlockSize())
328 .setReplication(file.getFileReplication());
329
330 AclFeature f = file.getAclFeature();
331 if (f != null) {
332 b.setAcl(buildAclEntries(f, state.getStringMap()));
333 }
334 return b;
335 }
336
337 public static INodeSection.INodeDirectory.Builder buildINodeDirectory(
338 INodeDirectoryAttributes dir, final SaverContext state) {
339 Quota.Counts quota = dir.getQuotaCounts();
340 INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory
341 .newBuilder().setModificationTime(dir.getModificationTime())
342 .setNsQuota(quota.get(Quota.NAMESPACE))
343 .setDsQuota(quota.get(Quota.DISKSPACE))
344 .setPermission(buildPermissionStatus(dir, state.getStringMap()));
345
346 AclFeature f = dir.getAclFeature();
347 if (f != null) {
348 b.setAcl(buildAclEntries(f, state.getStringMap()));
349 }
350 return b;
351 }
352
353 private final FSNamesystem fsn;
354 private final FileSummary.Builder summary;
355 private final SaveNamespaceContext context;
356 private final FSImageFormatProtobuf.Saver parent;
357
358 Saver(FSImageFormatProtobuf.Saver parent, FileSummary.Builder summary) {
359 this.parent = parent;
360 this.summary = summary;
361 this.context = parent.getContext();
362 this.fsn = context.getSourceNamesystem();
363 }
364
365 void serializeINodeDirectorySection(OutputStream out) throws IOException {
366 Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory()
367 .getINodeMap().getMapIterator();
368 final ArrayList<INodeReference> refList = parent.getSaverContext()
369 .getRefList();
370 int i = 0;
371 while (iter.hasNext()) {
372 INodeWithAdditionalFields n = iter.next();
373 if (!n.isDirectory()) {
374 continue;
375 }
376
377 ReadOnlyList<INode> children = n.asDirectory().getChildrenList(
378 Snapshot.CURRENT_STATE_ID);
379 if (children.size() > 0) {
380 INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection.
381 DirEntry.newBuilder().setParent(n.getId());
382 for (INode inode : children) {
383 if (!inode.isReference()) {
384 b.addChildren(inode.getId());
385 } else {
386 refList.add(inode.asReference());
387 b.addRefChildren(refList.size() - 1);
388 }
389 }
390 INodeDirectorySection.DirEntry e = b.build();
391 e.writeDelimitedTo(out);
392 }
393
394 ++i;
395 if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
396 context.checkCancelled();
397 }
398 }
399 parent.commitSection(summary,
400 FSImageFormatProtobuf.SectionName.INODE_DIR);
401 }
402
403 void serializeINodeSection(OutputStream out) throws IOException {
404 INodeMap inodesMap = fsn.dir.getINodeMap();
405
406 INodeSection.Builder b = INodeSection.newBuilder()
407 .setLastInodeId(fsn.getLastInodeId()).setNumInodes(inodesMap.size());
408 INodeSection s = b.build();
409 s.writeDelimitedTo(out);
410
411 int i = 0;
412 Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator();
413 while (iter.hasNext()) {
414 INodeWithAdditionalFields n = iter.next();
415 save(out, n);
416 ++i;
417 if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
418 context.checkCancelled();
419 }
420 }
421 parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE);
422 }
423
424 void serializeFilesUCSection(OutputStream out) throws IOException {
425 Map<String, INodeFile> ucMap = fsn.getFilesUnderConstruction();
426 for (Map.Entry<String, INodeFile> entry : ucMap.entrySet()) {
427 String path = entry.getKey();
428 INodeFile file = entry.getValue();
429 FileUnderConstructionEntry.Builder b = FileUnderConstructionEntry
430 .newBuilder().setInodeId(file.getId()).setFullPath(path);
431 FileUnderConstructionEntry e = b.build();
432 e.writeDelimitedTo(out);
433 }
434 parent.commitSection(summary,
435 FSImageFormatProtobuf.SectionName.FILES_UNDERCONSTRUCTION);
436 }
437
438 private void save(OutputStream out, INode n) throws IOException {
439 if (n.isDirectory()) {
440 save(out, n.asDirectory());
441 } else if (n.isFile()) {
442 save(out, n.asFile());
443 } else if (n.isSymlink()) {
444 save(out, n.asSymlink());
445 }
446 }
447
448 private void save(OutputStream out, INodeDirectory n) throws IOException {
449 INodeSection.INodeDirectory.Builder b = buildINodeDirectory(n,
450 parent.getSaverContext());
451 INodeSection.INode r = buildINodeCommon(n)
452 .setType(INodeSection.INode.Type.DIRECTORY).setDirectory(b).build();
453 r.writeDelimitedTo(out);
454 }
455
456 private void save(OutputStream out, INodeFile n) throws IOException {
457 INodeSection.INodeFile.Builder b = buildINodeFile(n,
458 parent.getSaverContext());
459
460 for (Block block : n.getBlocks()) {
461 b.addBlocks(PBHelper.convert(block));
462 }
463
464 FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature();
465 if (uc != null) {
466 INodeSection.FileUnderConstructionFeature f =
467 INodeSection.FileUnderConstructionFeature
468 .newBuilder().setClientName(uc.getClientName())
469 .setClientMachine(uc.getClientMachine()).build();
470 b.setFileUC(f);
471 }
472
473 INodeSection.INode r = buildINodeCommon(n)
474 .setType(INodeSection.INode.Type.FILE).setFile(b).build();
475 r.writeDelimitedTo(out);
476 }
477
478 private void save(OutputStream out, INodeSymlink n) throws IOException {
479 SaverContext state = parent.getSaverContext();
480 INodeSection.INodeSymlink.Builder b = INodeSection.INodeSymlink
481 .newBuilder()
482 .setPermission(buildPermissionStatus(n, state.getStringMap()))
483 .setTarget(ByteString.copyFrom(n.getSymlink()))
484 .setModificationTime(n.getModificationTime())
485 .setAccessTime(n.getAccessTime());
486
487 INodeSection.INode r = buildINodeCommon(n)
488 .setType(INodeSection.INode.Type.SYMLINK).setSymlink(b).build();
489 r.writeDelimitedTo(out);
490 }
491
492 private final INodeSection.INode.Builder buildINodeCommon(INode n) {
493 return INodeSection.INode.newBuilder()
494 .setId(n.getId())
495 .setName(ByteString.copyFrom(n.getLocalNameBytes()));
496 }
497 }
498
499 private FSImageFormatPBINode() {
500 }
501 }