001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hdfs.server.namenode;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.Iterator;
027import java.util.List;
028
029import org.apache.commons.logging.Log;
030import org.apache.commons.logging.LogFactory;
031import org.apache.hadoop.HadoopIllegalArgumentException;
032import org.apache.hadoop.classification.InterfaceAudience;
033import org.apache.hadoop.fs.permission.AclEntry;
034import org.apache.hadoop.fs.permission.AclEntryScope;
035import org.apache.hadoop.fs.permission.AclEntryType;
036import org.apache.hadoop.fs.permission.FsAction;
037import org.apache.hadoop.fs.permission.FsPermission;
038import org.apache.hadoop.fs.permission.PermissionStatus;
039import org.apache.hadoop.fs.StorageType;
040import org.apache.hadoop.fs.XAttr;
041import org.apache.hadoop.hdfs.protocol.Block;
042import org.apache.hadoop.hdfs.protocol.HdfsConstants;
043import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
044import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
045import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
046import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
047import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
048import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
049import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
050import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SaverContext;
051import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
052import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
053import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection;
054import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
055import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.AclFeatureProto;
056import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrCompactProto;
057import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrFeatureProto;
058import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeEntryProto;
059import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto;
060import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
061import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
062import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
063import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
064import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
065import org.apache.hadoop.hdfs.util.EnumCounters;
066import org.apache.hadoop.hdfs.util.ReadOnlyList;
067
068import com.google.common.base.Preconditions;
069import com.google.common.collect.ImmutableList;
070import com.google.protobuf.ByteString;
071
072@InterfaceAudience.Private
073public final class FSImageFormatPBINode {
074  private final static long USER_GROUP_STRID_MASK = (1 << 24) - 1;
075  private final static int USER_STRID_OFFSET = 40;
076  private final static int GROUP_STRID_OFFSET = 16;
077  private static final Log LOG = LogFactory.getLog(FSImageFormatPBINode.class);
078
079  public static final int ACL_ENTRY_NAME_MASK = (1 << 24) - 1;
080  public static final int ACL_ENTRY_NAME_OFFSET = 6;
081  public static final int ACL_ENTRY_TYPE_OFFSET = 3;
082  public static final int ACL_ENTRY_SCOPE_OFFSET = 5;
083  public static final int ACL_ENTRY_PERM_MASK = 7;
084  private static final int ACL_ENTRY_TYPE_MASK = 3;
085  private static final int ACL_ENTRY_SCOPE_MASK = 1;
086  private static final FsAction[] FSACTION_VALUES = FsAction.values();
087  private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES = AclEntryScope
088      .values();
089  private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType
090      .values();
091  
092  public static final int XATTR_NAMESPACE_MASK = 3;
093  public static final int XATTR_NAMESPACE_OFFSET = 30;
094  public static final int XATTR_NAME_MASK = (1 << 24) - 1;
095  public static final int XATTR_NAME_OFFSET = 6;
096
097  /* See the comments in fsimage.proto for an explanation of the following. */
098  public static final int XATTR_NAMESPACE_EXT_OFFSET = 5;
099  public static final int XATTR_NAMESPACE_EXT_MASK = 1;
100
101  private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
102      XAttr.NameSpace.values();
103  
104
105  public final static class Loader {
106    public static PermissionStatus loadPermission(long id,
107        final String[] stringTable) {
108      short perm = (short) (id & ((1 << GROUP_STRID_OFFSET) - 1));
109      int gsid = (int) ((id >> GROUP_STRID_OFFSET) & USER_GROUP_STRID_MASK);
110      int usid = (int) ((id >> USER_STRID_OFFSET) & USER_GROUP_STRID_MASK);
111      return new PermissionStatus(stringTable[usid], stringTable[gsid],
112          new FsPermission(perm));
113    }
114
115    public static ImmutableList<AclEntry> loadAclEntries(
116        AclFeatureProto proto, final String[] stringTable) {
117      ImmutableList.Builder<AclEntry> b = ImmutableList.builder();
118      for (int v : proto.getEntriesList()) {
119        int p = v & ACL_ENTRY_PERM_MASK;
120        int t = (v >> ACL_ENTRY_TYPE_OFFSET) & ACL_ENTRY_TYPE_MASK;
121        int s = (v >> ACL_ENTRY_SCOPE_OFFSET) & ACL_ENTRY_SCOPE_MASK;
122        int nid = (v >> ACL_ENTRY_NAME_OFFSET) & ACL_ENTRY_NAME_MASK;
123        String name = stringTable[nid];
124        b.add(new AclEntry.Builder().setName(name)
125            .setPermission(FSACTION_VALUES[p])
126            .setScope(ACL_ENTRY_SCOPE_VALUES[s])
127            .setType(ACL_ENTRY_TYPE_VALUES[t]).build());
128      }
129      return b.build();
130    }
131    
132    public static List<XAttr> loadXAttrs(
133        XAttrFeatureProto proto, final String[] stringTable) {
134      List<XAttr> b = new ArrayList<>();
135      for (XAttrCompactProto xAttrCompactProto : proto.getXAttrsList()) {
136        int v = xAttrCompactProto.getName();
137        int nid = (v >> XATTR_NAME_OFFSET) & XATTR_NAME_MASK;
138        int ns = (v >> XATTR_NAMESPACE_OFFSET) & XATTR_NAMESPACE_MASK;
139        ns |=
140            ((v >> XATTR_NAMESPACE_EXT_OFFSET) & XATTR_NAMESPACE_EXT_MASK) << 2;
141        String name = stringTable[nid];
142        byte[] value = null;
143        if (xAttrCompactProto.getValue() != null) {
144          value = xAttrCompactProto.getValue().toByteArray();
145        }
146        b.add(new XAttr.Builder().setNameSpace(XATTR_NAMESPACE_VALUES[ns])
147            .setName(name).setValue(value).build());
148      }
149      
150      return b;
151    }
152
153    public static ImmutableList<QuotaByStorageTypeEntry> loadQuotaByStorageTypeEntries(
154      QuotaByStorageTypeFeatureProto proto) {
155      ImmutableList.Builder<QuotaByStorageTypeEntry> b = ImmutableList.builder();
156      for (QuotaByStorageTypeEntryProto quotaEntry : proto.getQuotasList()) {
157        StorageType type = PBHelperClient.convertStorageType(quotaEntry.getStorageType());
158        long quota = quotaEntry.getQuota();
159        b.add(new QuotaByStorageTypeEntry.Builder().setStorageType(type)
160            .setQuota(quota).build());
161      }
162      return b.build();
163    }
164
165    public static INodeDirectory loadINodeDirectory(INodeSection.INode n,
166        LoaderContext state) {
167      assert n.getType() == INodeSection.INode.Type.DIRECTORY;
168      INodeSection.INodeDirectory d = n.getDirectory();
169
170      final PermissionStatus permissions = loadPermission(d.getPermission(),
171          state.getStringTable());
172      final INodeDirectory dir = new INodeDirectory(n.getId(), n.getName()
173          .toByteArray(), permissions, d.getModificationTime());
174      final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota();
175      if (nsQuota >= 0 || dsQuota >= 0) {
176        dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().
177            nameSpaceQuota(nsQuota).storageSpaceQuota(dsQuota).build());
178      }
179      EnumCounters<StorageType> typeQuotas = null;
180      if (d.hasTypeQuotas()) {
181        ImmutableList<QuotaByStorageTypeEntry> qes =
182            loadQuotaByStorageTypeEntries(d.getTypeQuotas());
183        typeQuotas = new EnumCounters<StorageType>(StorageType.class,
184            HdfsConstants.QUOTA_RESET);
185        for (QuotaByStorageTypeEntry qe : qes) {
186          if (qe.getQuota() >= 0 && qe.getStorageType() != null &&
187              qe.getStorageType().supportTypeQuota()) {
188            typeQuotas.set(qe.getStorageType(), qe.getQuota());
189          }
190        }
191
192        if (typeQuotas.anyGreaterOrEqual(0)) {
193          DirectoryWithQuotaFeature q = dir.getDirectoryWithQuotaFeature();
194          if (q == null) {
195            dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.
196                Builder().typeQuotas(typeQuotas).build());
197          } else {
198            q.setQuota(typeQuotas);
199          }
200        }
201      }
202
203      if (d.hasAcl()) {
204        int[] entries = AclEntryStatusFormat.toInt(loadAclEntries(
205            d.getAcl(), state.getStringTable()));
206        dir.addAclFeature(new AclFeature(entries));
207      }
208      if (d.hasXAttrs()) {
209        dir.addXAttrFeature(new XAttrFeature(
210            loadXAttrs(d.getXAttrs(), state.getStringTable())));
211      }
212      return dir;
213    }
214
215    public static void updateBlocksMap(INodeFile file, BlockManager bm) {
216      // Add file->block mapping
217      final BlockInfo[] blocks = file.getBlocks();
218      if (blocks != null) {
219        for (int i = 0; i < blocks.length; i++) {
220          file.setBlock(i, bm.addBlockCollection(blocks[i], file));
221        }
222      }
223    }
224
225    private final FSDirectory dir;
226    private final FSNamesystem fsn;
227    private final FSImageFormatProtobuf.Loader parent;
228
229    Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) {
230      this.fsn = fsn;
231      this.dir = fsn.dir;
232      this.parent = parent;
233    }
234
235    void loadINodeDirectorySection(InputStream in) throws IOException {
236      final List<INodeReference> refList = parent.getLoaderContext()
237          .getRefList();
238      while (true) {
239        INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry
240            .parseDelimitedFrom(in);
241        // note that in is a LimitedInputStream
242        if (e == null) {
243          break;
244        }
245        INodeDirectory p = dir.getInode(e.getParent()).asDirectory();
246        for (long id : e.getChildrenList()) {
247          INode child = dir.getInode(id);
248          addToParent(p, child);
249        }
250        for (int refId : e.getRefChildrenList()) {
251          INodeReference ref = refList.get(refId);
252          addToParent(p, ref);
253        }
254      }
255    }
256
257    void loadINodeSection(InputStream in, StartupProgress prog,
258        Step currentStep) throws IOException {
259      INodeSection s = INodeSection.parseDelimitedFrom(in);
260      fsn.dir.resetLastInodeId(s.getLastInodeId());
261      long numInodes = s.getNumInodes();
262      LOG.info("Loading " + numInodes + " INodes.");
263      prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, numInodes);
264      Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep);
265      for (int i = 0; i < numInodes; ++i) {
266        INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in);
267        if (p.getId() == INodeId.ROOT_INODE_ID) {
268          loadRootINode(p);
269        } else {
270          INode n = loadINode(p);
271          dir.addToInodeMap(n);
272        }
273        counter.increment();
274      }
275    }
276
277    /**
278     * Load the under-construction files section, and update the lease map
279     */
280    void loadFilesUnderConstructionSection(InputStream in) throws IOException {
281      // Leases are added when the inode section is loaded. This section is
282      // still read in for compatibility reasons.
283      while (true) {
284        FileUnderConstructionEntry entry = FileUnderConstructionEntry
285            .parseDelimitedFrom(in);
286        if (entry == null) {
287          break;
288        }
289      }
290    }
291
292    private void addToParent(INodeDirectory parent, INode child) {
293      if (parent == dir.rootDir && FSDirectory.isReservedName(child)) {
294        throw new HadoopIllegalArgumentException("File name \""
295            + child.getLocalName() + "\" is reserved. Please "
296            + " change the name of the existing file or directory to another "
297            + "name before upgrading to this release.");
298      }
299      // NOTE: This does not update space counts for parents
300      if (!parent.addChild(child)) {
301        return;
302      }
303      dir.cacheName(child);
304
305      if (child.isFile()) {
306        updateBlocksMap(child.asFile(), fsn.getBlockManager());
307      }
308    }
309
310    private INode loadINode(INodeSection.INode n) {
311      switch (n.getType()) {
312      case FILE:
313        return loadINodeFile(n);
314      case DIRECTORY:
315        return loadINodeDirectory(n, parent.getLoaderContext());
316      case SYMLINK:
317        return loadINodeSymlink(n);
318      default:
319        break;
320      }
321      return null;
322    }
323
324    private INodeFile loadINodeFile(INodeSection.INode n) {
325      assert n.getType() == INodeSection.INode.Type.FILE;
326      INodeSection.INodeFile f = n.getFile();
327      List<BlockProto> bp = f.getBlocksList();
328      short replication = (short) f.getReplication();
329      LoaderContext state = parent.getLoaderContext();
330
331      BlockInfo[] blocks = new BlockInfo[bp.size()];
332      for (int i = 0, e = bp.size(); i < e; ++i) {
333        blocks[i] =
334            new BlockInfoContiguous(PBHelperClient.convert(bp.get(i)), replication);
335      }
336      final PermissionStatus permissions = loadPermission(f.getPermission(),
337          parent.getLoaderContext().getStringTable());
338
339      final INodeFile file = new INodeFile(n.getId(),
340          n.getName().toByteArray(), permissions, f.getModificationTime(),
341          f.getAccessTime(), blocks, replication, f.getPreferredBlockSize(),
342          (byte)f.getStoragePolicyID());
343
344      if (f.hasAcl()) {
345        int[] entries = AclEntryStatusFormat.toInt(loadAclEntries(
346            f.getAcl(), state.getStringTable()));
347        file.addAclFeature(new AclFeature(entries));
348      }
349      
350      if (f.hasXAttrs()) {
351        file.addXAttrFeature(new XAttrFeature(
352            loadXAttrs(f.getXAttrs(), state.getStringTable())));
353      }
354
355      // under-construction information
356      if (f.hasFileUC()) {
357        INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
358        file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
359        // update the lease manager
360        fsn.leaseManager.addLease(uc.getClientName(), file.getId());
361        if (blocks.length > 0) {
362          BlockInfo lastBlk = file.getLastBlock();
363          lastBlk.convertToBlockUnderConstruction(
364              HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, null);
365        }
366      }
367      return file;
368    }
369
370
371    private INodeSymlink loadINodeSymlink(INodeSection.INode n) {
372      assert n.getType() == INodeSection.INode.Type.SYMLINK;
373      INodeSection.INodeSymlink s = n.getSymlink();
374      final PermissionStatus permissions = loadPermission(s.getPermission(),
375          parent.getLoaderContext().getStringTable());
376      INodeSymlink sym = new INodeSymlink(n.getId(), n.getName().toByteArray(),
377          permissions, s.getModificationTime(), s.getAccessTime(),
378          s.getTarget().toStringUtf8());
379      return sym;
380    }
381
382    private void loadRootINode(INodeSection.INode p) {
383      INodeDirectory root = loadINodeDirectory(p, parent.getLoaderContext());
384      final QuotaCounts q = root.getQuotaCounts();
385      final long nsQuota = q.getNameSpace();
386      final long dsQuota = q.getStorageSpace();
387      if (nsQuota != -1 || dsQuota != -1) {
388        dir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
389      }
390      final EnumCounters<StorageType> typeQuotas = q.getTypeSpaces();
391      if (typeQuotas.anyGreaterOrEqual(0)) {
392        dir.rootDir.getDirectoryWithQuotaFeature().setQuota(typeQuotas);
393      }
394      dir.rootDir.cloneModificationTime(root);
395      dir.rootDir.clonePermissionStatus(root);
396      final AclFeature af = root.getFeature(AclFeature.class);
397      if (af != null) {
398        dir.rootDir.addAclFeature(af);
399      }
400      // root dir supports having extended attributes according to POSIX
401      final XAttrFeature f = root.getXAttrFeature();
402      if (f != null) {
403        dir.rootDir.addXAttrFeature(f);
404      }
405      dir.addRootDirToEncryptionZone(f);
406    }
407  }
408
409  public final static class Saver {
410    private static long buildPermissionStatus(INodeAttributes n,
411        final SaverContext.DeduplicationMap<String> stringMap) {
412      long userId = stringMap.getId(n.getUserName());
413      long groupId = stringMap.getId(n.getGroupName());
414      return ((userId & USER_GROUP_STRID_MASK) << USER_STRID_OFFSET)
415          | ((groupId & USER_GROUP_STRID_MASK) << GROUP_STRID_OFFSET)
416          | n.getFsPermissionShort();
417    }
418
419    private static AclFeatureProto.Builder buildAclEntries(AclFeature f,
420        final SaverContext.DeduplicationMap<String> map) {
421      AclFeatureProto.Builder b = AclFeatureProto.newBuilder();
422      for (int pos = 0, e; pos < f.getEntriesSize(); pos++) {
423        e = f.getEntryAt(pos);
424        int nameId = map.getId(AclEntryStatusFormat.getName(e));
425        int v = ((nameId & ACL_ENTRY_NAME_MASK) << ACL_ENTRY_NAME_OFFSET)
426            | (AclEntryStatusFormat.getType(e).ordinal() << ACL_ENTRY_TYPE_OFFSET)
427            | (AclEntryStatusFormat.getScope(e).ordinal() << ACL_ENTRY_SCOPE_OFFSET)
428            | (AclEntryStatusFormat.getPermission(e).ordinal());
429        b.addEntries(v);
430      }
431      return b;
432    }
433    
434    private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f,
435        final SaverContext.DeduplicationMap<String> stringMap) {
436      XAttrFeatureProto.Builder b = XAttrFeatureProto.newBuilder();
437      for (XAttr a : f.getXAttrs()) {
438        XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto.
439            newBuilder();
440        int nsOrd = a.getNameSpace().ordinal();
441        Preconditions.checkArgument(nsOrd < 8, "Too many namespaces.");
442        int v = ((nsOrd & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET)
443            | ((stringMap.getId(a.getName()) & XATTR_NAME_MASK) <<
444                XATTR_NAME_OFFSET);
445        v |= (((nsOrd >> 2) & XATTR_NAMESPACE_EXT_MASK) <<
446            XATTR_NAMESPACE_EXT_OFFSET);
447        xAttrCompactBuilder.setName(v);
448        if (a.getValue() != null) {
449          xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue()));
450        }
451        b.addXAttrs(xAttrCompactBuilder.build());
452      }
453      
454      return b;
455    }
456
457    private static QuotaByStorageTypeFeatureProto.Builder
458        buildQuotaByStorageTypeEntries(QuotaCounts q) {
459      QuotaByStorageTypeFeatureProto.Builder b =
460          QuotaByStorageTypeFeatureProto.newBuilder();
461      for (StorageType t: StorageType.getTypesSupportingQuota()) {
462        if (q.getTypeSpace(t) >= 0) {
463          QuotaByStorageTypeEntryProto.Builder eb =
464              QuotaByStorageTypeEntryProto.newBuilder().
465              setStorageType(PBHelperClient.convertStorageType(t)).
466              setQuota(q.getTypeSpace(t));
467          b.addQuotas(eb);
468        }
469      }
470      return b;
471    }
472
473    public static INodeSection.INodeFile.Builder buildINodeFile(
474        INodeFileAttributes file, final SaverContext state) {
475      INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder()
476          .setAccessTime(file.getAccessTime())
477          .setModificationTime(file.getModificationTime())
478          .setPermission(buildPermissionStatus(file, state.getStringMap()))
479          .setPreferredBlockSize(file.getPreferredBlockSize())
480          .setReplication(file.getFileReplication())
481          .setStoragePolicyID(file.getLocalStoragePolicyID());
482
483      AclFeature f = file.getAclFeature();
484      if (f != null) {
485        b.setAcl(buildAclEntries(f, state.getStringMap()));
486      }
487      XAttrFeature xAttrFeature = file.getXAttrFeature();
488      if (xAttrFeature != null) {
489        b.setXAttrs(buildXAttrs(xAttrFeature, state.getStringMap()));
490      }
491      return b;
492    }
493
494    public static INodeSection.INodeDirectory.Builder buildINodeDirectory(
495        INodeDirectoryAttributes dir, final SaverContext state) {
496      QuotaCounts quota = dir.getQuotaCounts();
497      INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory
498          .newBuilder().setModificationTime(dir.getModificationTime())
499          .setNsQuota(quota.getNameSpace())
500          .setDsQuota(quota.getStorageSpace())
501          .setPermission(buildPermissionStatus(dir, state.getStringMap()));
502
503      if (quota.getTypeSpaces().anyGreaterOrEqual(0)) {
504        b.setTypeQuotas(buildQuotaByStorageTypeEntries(quota));
505      }
506
507      AclFeature f = dir.getAclFeature();
508      if (f != null) {
509        b.setAcl(buildAclEntries(f, state.getStringMap()));
510      }
511      XAttrFeature xAttrFeature = dir.getXAttrFeature();
512      if (xAttrFeature != null) {
513        b.setXAttrs(buildXAttrs(xAttrFeature, state.getStringMap()));
514      }
515      return b;
516    }
517
518    private final FSNamesystem fsn;
519    private final FileSummary.Builder summary;
520    private final SaveNamespaceContext context;
521    private final FSImageFormatProtobuf.Saver parent;
522
523    Saver(FSImageFormatProtobuf.Saver parent, FileSummary.Builder summary) {
524      this.parent = parent;
525      this.summary = summary;
526      this.context = parent.getContext();
527      this.fsn = context.getSourceNamesystem();
528    }
529
530    void serializeINodeDirectorySection(OutputStream out) throws IOException {
531      Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory()
532          .getINodeMap().getMapIterator();
533      final ArrayList<INodeReference> refList = parent.getSaverContext()
534          .getRefList();
535      int i = 0;
536      while (iter.hasNext()) {
537        INodeWithAdditionalFields n = iter.next();
538        if (!n.isDirectory()) {
539          continue;
540        }
541
542        ReadOnlyList<INode> children = n.asDirectory().getChildrenList(
543            Snapshot.CURRENT_STATE_ID);
544        if (children.size() > 0) {
545          INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection.
546              DirEntry.newBuilder().setParent(n.getId());
547          for (INode inode : children) {
548            if (!inode.isReference()) {
549              b.addChildren(inode.getId());
550            } else {
551              refList.add(inode.asReference());
552              b.addRefChildren(refList.size() - 1);
553            }
554          }
555          INodeDirectorySection.DirEntry e = b.build();
556          e.writeDelimitedTo(out);
557        }
558
559        ++i;
560        if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
561          context.checkCancelled();
562        }
563      }
564      parent.commitSection(summary,
565          FSImageFormatProtobuf.SectionName.INODE_DIR);
566    }
567
568    void serializeINodeSection(OutputStream out) throws IOException {
569      INodeMap inodesMap = fsn.dir.getINodeMap();
570
571      INodeSection.Builder b = INodeSection.newBuilder()
572          .setLastInodeId(fsn.dir.getLastInodeId()).setNumInodes(inodesMap.size());
573      INodeSection s = b.build();
574      s.writeDelimitedTo(out);
575
576      int i = 0;
577      Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator();
578      while (iter.hasNext()) {
579        INodeWithAdditionalFields n = iter.next();
580        save(out, n);
581        ++i;
582        if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
583          context.checkCancelled();
584        }
585      }
586      parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE);
587    }
588
589    void serializeFilesUCSection(OutputStream out) throws IOException {
590      Collection<Long> filesWithUC = fsn.getLeaseManager()
591              .getINodeIdWithLeases();
592      for (Long id : filesWithUC) {
593        INode inode = fsn.getFSDirectory().getInode(id);
594        if (inode == null) {
595          LOG.warn("Fail to find inode " + id + " when saving the leases.");
596          continue;
597        }
598        INodeFile file = inode.asFile();
599        if (!file.isUnderConstruction()) {
600          LOG.warn("Fail to save the lease for inode id " + id
601                       + " as the file is not under construction");
602          continue;
603        }
604        String path = file.getFullPathName();
605        FileUnderConstructionEntry.Builder b = FileUnderConstructionEntry
606            .newBuilder().setInodeId(file.getId()).setFullPath(path);
607        FileUnderConstructionEntry e = b.build();
608        e.writeDelimitedTo(out);
609      }
610      parent.commitSection(summary,
611          FSImageFormatProtobuf.SectionName.FILES_UNDERCONSTRUCTION);
612    }
613
614    private void save(OutputStream out, INode n) throws IOException {
615      if (n.isDirectory()) {
616        save(out, n.asDirectory());
617      } else if (n.isFile()) {
618        save(out, n.asFile());
619      } else if (n.isSymlink()) {
620        save(out, n.asSymlink());
621      }
622    }
623
624    private void save(OutputStream out, INodeDirectory n) throws IOException {
625      INodeSection.INodeDirectory.Builder b = buildINodeDirectory(n,
626          parent.getSaverContext());
627      INodeSection.INode r = buildINodeCommon(n)
628          .setType(INodeSection.INode.Type.DIRECTORY).setDirectory(b).build();
629      r.writeDelimitedTo(out);
630    }
631
632    private void save(OutputStream out, INodeFile n) throws IOException {
633      INodeSection.INodeFile.Builder b = buildINodeFile(n,
634          parent.getSaverContext());
635
636      if (n.getBlocks() != null) {
637        for (Block block : n.getBlocks()) {
638          b.addBlocks(PBHelperClient.convert(block));
639        }
640      }
641
642      FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature();
643      if (uc != null) {
644        INodeSection.FileUnderConstructionFeature f =
645            INodeSection.FileUnderConstructionFeature
646            .newBuilder().setClientName(uc.getClientName())
647            .setClientMachine(uc.getClientMachine()).build();
648        b.setFileUC(f);
649      }
650
651      INodeSection.INode r = buildINodeCommon(n)
652          .setType(INodeSection.INode.Type.FILE).setFile(b).build();
653      r.writeDelimitedTo(out);
654    }
655
656    private void save(OutputStream out, INodeSymlink n) throws IOException {
657      SaverContext state = parent.getSaverContext();
658      INodeSection.INodeSymlink.Builder b = INodeSection.INodeSymlink
659          .newBuilder()
660          .setPermission(buildPermissionStatus(n, state.getStringMap()))
661          .setTarget(ByteString.copyFrom(n.getSymlink()))
662          .setModificationTime(n.getModificationTime())
663          .setAccessTime(n.getAccessTime());
664
665      INodeSection.INode r = buildINodeCommon(n)
666          .setType(INodeSection.INode.Type.SYMLINK).setSymlink(b).build();
667      r.writeDelimitedTo(out);
668    }
669
670    private final INodeSection.INode.Builder buildINodeCommon(INode n) {
671      return INodeSection.INode.newBuilder()
672          .setId(n.getId())
673          .setName(ByteString.copyFrom(n.getLocalNameBytes()));
674    }
675  }
676
677  private FSImageFormatPBINode() {
678  }
679}