001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.blockmanagement;
019
020 import org.apache.hadoop.classification.InterfaceAudience;
021 import org.apache.hadoop.hdfs.protocol.Block;
022 import org.apache.hadoop.hdfs.server.namenode.NameNode;
023 import org.apache.hadoop.ipc.Server;
024
025 import java.util.*;
026
027 /**
028 * Stores information about all corrupt blocks in the File System.
029 * A Block is considered corrupt only if all of its replicas are
030 * corrupt. While reporting replicas of a Block, we hide any corrupt
031 * copies. These copies are removed once Block is found to have
032 * expected number of good replicas.
033 * Mapping: Block -> TreeSet<DatanodeDescriptor>
034 */
035
036 @InterfaceAudience.Private
037 public class CorruptReplicasMap{
038
039 /** The corruption reason code */
040 public static enum Reason {
041 NONE, // not specified.
042 ANY, // wildcard reason
043 GENSTAMP_MISMATCH, // mismatch in generation stamps
044 SIZE_MISMATCH, // mismatch in sizes
045 INVALID_STATE, // invalid state
046 CORRUPTION_REPORTED // client or datanode reported the corruption
047 }
048
049 private final SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
050 new TreeMap<Block, Map<DatanodeDescriptor, Reason>>();
051
052 /**
053 * Mark the block belonging to datanode as corrupt.
054 *
055 * @param blk Block to be added to CorruptReplicasMap
056 * @param dn DatanodeDescriptor which holds the corrupt replica
057 * @param reason a textual reason (for logging purposes)
058 * @param reasonCode the enum representation of the reason
059 */
060 void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
061 String reason, Reason reasonCode) {
062 Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
063 if (nodes == null) {
064 nodes = new HashMap<DatanodeDescriptor, Reason>();
065 corruptReplicasMap.put(blk, nodes);
066 }
067
068 String reasonText;
069 if (reason != null) {
070 reasonText = " because " + reason;
071 } else {
072 reasonText = "";
073 }
074
075 if (!nodes.keySet().contains(dn)) {
076 NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
077 blk.getBlockName() +
078 " added as corrupt on " + dn +
079 " by " + Server.getRemoteIp() +
080 reasonText);
081 } else {
082 NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
083 "duplicate requested for " +
084 blk.getBlockName() + " to add as corrupt " +
085 "on " + dn +
086 " by " + Server.getRemoteIp() +
087 reasonText);
088 }
089 // Add the node or update the reason.
090 nodes.put(dn, reasonCode);
091 }
092
093 /**
094 * Remove Block from CorruptBlocksMap
095 *
096 * @param blk Block to be removed
097 */
098 void removeFromCorruptReplicasMap(Block blk) {
099 if (corruptReplicasMap != null) {
100 corruptReplicasMap.remove(blk);
101 }
102 }
103
104 /**
105 * Remove the block at the given datanode from CorruptBlockMap
106 * @param blk block to be removed
107 * @param datanode datanode where the block is located
108 * @return true if the removal is successful;
109 false if the replica is not in the map
110 */
111 boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode) {
112 return removeFromCorruptReplicasMap(blk, datanode, Reason.ANY);
113 }
114
115 boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode,
116 Reason reason) {
117 Map <DatanodeDescriptor, Reason> datanodes = corruptReplicasMap.get(blk);
118 if (datanodes==null)
119 return false;
120
121 // if reasons can be compared but don't match, return false.
122 Reason storedReason = datanodes.get(datanode);
123 if (reason != Reason.ANY && storedReason != null &&
124 reason != storedReason) {
125 return false;
126 }
127
128 if (datanodes.remove(datanode) != null) { // remove the replicas
129 if (datanodes.isEmpty()) {
130 // remove the block if there is no more corrupted replicas
131 corruptReplicasMap.remove(blk);
132 }
133 return true;
134 }
135 return false;
136 }
137
138
139 /**
140 * Get Nodes which have corrupt replicas of Block
141 *
142 * @param blk Block for which nodes are requested
143 * @return collection of nodes. Null if does not exists
144 */
145 Collection<DatanodeDescriptor> getNodes(Block blk) {
146 Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
147 if (nodes == null)
148 return null;
149 return nodes.keySet();
150 }
151
152 /**
153 * Check if replica belonging to Datanode is corrupt
154 *
155 * @param blk Block to check
156 * @param node DatanodeDescriptor which holds the replica
157 * @return true if replica is corrupt, false if does not exists in this map
158 */
159 boolean isReplicaCorrupt(Block blk, DatanodeDescriptor node) {
160 Collection<DatanodeDescriptor> nodes = getNodes(blk);
161 return ((nodes != null) && (nodes.contains(node)));
162 }
163
164 int numCorruptReplicas(Block blk) {
165 Collection<DatanodeDescriptor> nodes = getNodes(blk);
166 return (nodes == null) ? 0 : nodes.size();
167 }
168
169 int size() {
170 return corruptReplicasMap.size();
171 }
172
173 /**
174 * Return a range of corrupt replica block ids. Up to numExpectedBlocks
175 * blocks starting at the next block after startingBlockId are returned
176 * (fewer if numExpectedBlocks blocks are unavailable). If startingBlockId
177 * is null, up to numExpectedBlocks blocks are returned from the beginning.
178 * If startingBlockId cannot be found, null is returned.
179 *
180 * @param numExpectedBlocks Number of block ids to return.
181 * 0 <= numExpectedBlocks <= 100
182 * @param startingBlockId Block id from which to start. If null, start at
183 * beginning.
184 * @return Up to numExpectedBlocks blocks from startingBlockId if it exists
185 *
186 */
187 long[] getCorruptReplicaBlockIds(int numExpectedBlocks,
188 Long startingBlockId) {
189 if (numExpectedBlocks < 0 || numExpectedBlocks > 100) {
190 return null;
191 }
192
193 Iterator<Block> blockIt = corruptReplicasMap.keySet().iterator();
194
195 // if the starting block id was specified, iterate over keys until
196 // we find the matching block. If we find a matching block, break
197 // to leave the iterator on the next block after the specified block.
198 if (startingBlockId != null) {
199 boolean isBlockFound = false;
200 while (blockIt.hasNext()) {
201 Block b = blockIt.next();
202 if (b.getBlockId() == startingBlockId) {
203 isBlockFound = true;
204 break;
205 }
206 }
207
208 if (!isBlockFound) {
209 return null;
210 }
211 }
212
213 ArrayList<Long> corruptReplicaBlockIds = new ArrayList<Long>();
214
215 // append up to numExpectedBlocks blockIds to our list
216 for(int i=0; i<numExpectedBlocks && blockIt.hasNext(); i++) {
217 corruptReplicaBlockIds.add(blockIt.next().getBlockId());
218 }
219
220 long[] ret = new long[corruptReplicaBlockIds.size()];
221 for(int i=0; i<ret.length; i++) {
222 ret[i] = corruptReplicaBlockIds.get(i);
223 }
224
225 return ret;
226 }
227 }