001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.blockmanagement;
019
020 import org.apache.hadoop.classification.InterfaceAudience;
021 import org.apache.hadoop.hdfs.protocol.Block;
022 import org.apache.hadoop.hdfs.server.namenode.NameNode;
023 import org.apache.hadoop.ipc.Server;
024
025 import java.util.*;
026
027 /**
028 * Stores information about all corrupt blocks in the File System.
029 * A Block is considered corrupt only if all of its replicas are
030 * corrupt. While reporting replicas of a Block, we hide any corrupt
031 * copies. These copies are removed once Block is found to have
032 * expected number of good replicas.
033 * Mapping: Block -> TreeSet<DatanodeDescriptor>
034 */
035
036 @InterfaceAudience.Private
037 public class CorruptReplicasMap{
038
039 /** The corruption reason code */
040 public static enum Reason {
041 NONE, // not specified.
042 ANY, // wildcard reason
043 GENSTAMP_MISMATCH, // mismatch in generation stamps
044 SIZE_MISMATCH, // mismatch in sizes
045 INVALID_STATE, // invalid state
046 CORRUPTION_REPORTED // client or datanode reported the corruption
047 }
048
049 private final SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
050 new TreeMap<Block, Map<DatanodeDescriptor, Reason>>();
051
052 /**
053 * Mark the block belonging to datanode as corrupt.
054 *
055 * @param blk Block to be added to CorruptReplicasMap
056 * @param dn DatanodeDescriptor which holds the corrupt replica
057 * @param reason a textual reason (for logging purposes)
058 */
059 public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
060 String reason) {
061 addToCorruptReplicasMap(blk, dn, reason, Reason.NONE);
062 }
063
064 /**
065 * Mark the block belonging to datanode as corrupt.
066 *
067 * @param blk Block to be added to CorruptReplicasMap
068 * @param dn DatanodeDescriptor which holds the corrupt replica
069 * @param reason a textual reason (for logging purposes)
070 * @param reasonCode the enum representation of the reason
071 */
072 public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
073 String reason, Reason reasonCode) {
074 Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
075 if (nodes == null) {
076 nodes = new HashMap<DatanodeDescriptor, Reason>();
077 corruptReplicasMap.put(blk, nodes);
078 }
079
080 String reasonText;
081 if (reason != null) {
082 reasonText = " because " + reason;
083 } else {
084 reasonText = "";
085 }
086
087 if (!nodes.keySet().contains(dn)) {
088 NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
089 blk.getBlockName() +
090 " added as corrupt on " + dn +
091 " by " + Server.getRemoteIp() +
092 reasonText);
093 } else {
094 NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
095 "duplicate requested for " +
096 blk.getBlockName() + " to add as corrupt " +
097 "on " + dn +
098 " by " + Server.getRemoteIp() +
099 reasonText);
100 }
101 // Add the node or update the reason.
102 nodes.put(dn, reasonCode);
103 }
104
105 /**
106 * Remove Block from CorruptBlocksMap
107 *
108 * @param blk Block to be removed
109 */
110 void removeFromCorruptReplicasMap(Block blk) {
111 if (corruptReplicasMap != null) {
112 corruptReplicasMap.remove(blk);
113 }
114 }
115
116 /**
117 * Remove the block at the given datanode from CorruptBlockMap
118 * @param blk block to be removed
119 * @param datanode datanode where the block is located
120 * @return true if the removal is successful;
121 false if the replica is not in the map
122 */
123 boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode) {
124 return removeFromCorruptReplicasMap(blk, datanode, Reason.ANY);
125 }
126
127 boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode,
128 Reason reason) {
129 Map <DatanodeDescriptor, Reason> datanodes = corruptReplicasMap.get(blk);
130 boolean removed = false;
131 if (datanodes==null)
132 return false;
133
134 // if reasons can be compared but don't match, return false.
135 Reason storedReason = datanodes.get(datanode);
136 if (reason != Reason.ANY && storedReason != null &&
137 reason != storedReason) {
138 return false;
139 }
140
141 if (datanodes.remove(datanode) != null) { // remove the replicas
142 if (datanodes.isEmpty()) {
143 // remove the block if there is no more corrupted replicas
144 corruptReplicasMap.remove(blk);
145 }
146 return true;
147 }
148 return false;
149 }
150
151
152 /**
153 * Get Nodes which have corrupt replicas of Block
154 *
155 * @param blk Block for which nodes are requested
156 * @return collection of nodes. Null if does not exists
157 */
158 Collection<DatanodeDescriptor> getNodes(Block blk) {
159 Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
160 if (nodes == null)
161 return null;
162 return nodes.keySet();
163 }
164
165 /**
166 * Check if replica belonging to Datanode is corrupt
167 *
168 * @param blk Block to check
169 * @param node DatanodeDescriptor which holds the replica
170 * @return true if replica is corrupt, false if does not exists in this map
171 */
172 boolean isReplicaCorrupt(Block blk, DatanodeDescriptor node) {
173 Collection<DatanodeDescriptor> nodes = getNodes(blk);
174 return ((nodes != null) && (nodes.contains(node)));
175 }
176
177 public int numCorruptReplicas(Block blk) {
178 Collection<DatanodeDescriptor> nodes = getNodes(blk);
179 return (nodes == null) ? 0 : nodes.size();
180 }
181
182 public int size() {
183 return corruptReplicasMap.size();
184 }
185
186 /**
187 * Return a range of corrupt replica block ids. Up to numExpectedBlocks
188 * blocks starting at the next block after startingBlockId are returned
189 * (fewer if numExpectedBlocks blocks are unavailable). If startingBlockId
190 * is null, up to numExpectedBlocks blocks are returned from the beginning.
191 * If startingBlockId cannot be found, null is returned.
192 *
193 * @param numExpectedBlocks Number of block ids to return.
194 * 0 <= numExpectedBlocks <= 100
195 * @param startingBlockId Block id from which to start. If null, start at
196 * beginning.
197 * @return Up to numExpectedBlocks blocks from startingBlockId if it exists
198 *
199 */
200 long[] getCorruptReplicaBlockIds(int numExpectedBlocks,
201 Long startingBlockId) {
202 if (numExpectedBlocks < 0 || numExpectedBlocks > 100) {
203 return null;
204 }
205
206 Iterator<Block> blockIt = corruptReplicasMap.keySet().iterator();
207
208 // if the starting block id was specified, iterate over keys until
209 // we find the matching block. If we find a matching block, break
210 // to leave the iterator on the next block after the specified block.
211 if (startingBlockId != null) {
212 boolean isBlockFound = false;
213 while (blockIt.hasNext()) {
214 Block b = blockIt.next();
215 if (b.getBlockId() == startingBlockId) {
216 isBlockFound = true;
217 break;
218 }
219 }
220
221 if (!isBlockFound) {
222 return null;
223 }
224 }
225
226 ArrayList<Long> corruptReplicaBlockIds = new ArrayList<Long>();
227
228 // append up to numExpectedBlocks blockIds to our list
229 for(int i=0; i<numExpectedBlocks && blockIt.hasNext(); i++) {
230 corruptReplicaBlockIds.add(blockIt.next().getBlockId());
231 }
232
233 long[] ret = new long[corruptReplicaBlockIds.size()];
234 for(int i=0; i<ret.length; i++) {
235 ret[i] = corruptReplicaBlockIds.get(i);
236 }
237
238 return ret;
239 }
240 }