001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.datanode.fsdataset;
019
020
021 import java.io.File;
022 import java.io.IOException;
023 import java.io.InputStream;
024 import java.util.List;
025 import java.util.Map;
026
027 import org.apache.hadoop.classification.InterfaceAudience;
028 import org.apache.hadoop.conf.Configuration;
029 import org.apache.hadoop.hdfs.DFSConfigKeys;
030 import org.apache.hadoop.hdfs.protocol.Block;
031 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
032 import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
033 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
034 import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
035 import org.apache.hadoop.hdfs.server.datanode.DataNode;
036 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
037 import org.apache.hadoop.hdfs.server.datanode.Replica;
038 import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
039 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
040 import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
041 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
042 import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
043 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
044 import org.apache.hadoop.util.ReflectionUtils;
045
046 /**
047 * This is a service provider interface for the underlying storage that
048 * stores replicas for a data node.
049 * The default implementation stores replicas on local drives.
050 */
051 @InterfaceAudience.Private
052 public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
053 /**
054 * A factory for creating {@link FsDatasetSpi} objects.
055 */
056 public static abstract class Factory<D extends FsDatasetSpi<?>> {
057 /** @return the configured factory. */
058 public static Factory<?> getFactory(Configuration conf) {
059 @SuppressWarnings("rawtypes")
060 final Class<? extends Factory> clazz = conf.getClass(
061 DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY,
062 FsDatasetFactory.class,
063 Factory.class);
064 return ReflectionUtils.newInstance(clazz, conf);
065 }
066
067 /** Create a new object. */
068 public abstract D newInstance(DataNode datanode, DataStorage storage,
069 Configuration conf) throws IOException;
070
071 /** Does the factory create simulated objects? */
072 public boolean isSimulated() {
073 return false;
074 }
075 }
076
077 /**
078 * Create rolling logs.
079 *
080 * @param prefix the prefix of the log names.
081 * @return rolling logs
082 */
083 public RollingLogs createRollingLogs(String bpid, String prefix
084 ) throws IOException;
085
086 /** @return a list of volumes. */
087 public List<V> getVolumes();
088
089 /** @return the volume that contains a replica of the block. */
090 public V getVolume(ExtendedBlock b);
091
092 /** @return a volume information map (name => info). */
093 public Map<String, Object> getVolumeInfoMap();
094
095 /** @return a list of block pools. */
096 public String[] getBlockPoolList();
097
098 /** @return a list of finalized blocks for the given block pool. */
099 public List<Block> getFinalizedBlocks(String bpid);
100
101 /**
102 * Check whether the in-memory block record matches the block on the disk,
103 * and, in case that they are not matched, update the record or mark it
104 * as corrupted.
105 */
106 public void checkAndUpdate(String bpid, long blockId, File diskFile,
107 File diskMetaFile, FsVolumeSpi vol);
108
109 /**
110 * @param b - the block
111 * @return a stream if the meta-data of the block exists;
112 * otherwise, return null.
113 * @throws IOException
114 */
115 public LengthInputStream getMetaDataInputStream(ExtendedBlock b
116 ) throws IOException;
117
118 /**
119 * Returns the specified block's on-disk length (excluding metadata)
120 * @param b
121 * @return the specified block's on-disk length (excluding metadta)
122 * @throws IOException
123 */
124 public long getLength(ExtendedBlock b) throws IOException;
125
126 /**
127 * Get reference to the replica meta info in the replicasMap.
128 * To be called from methods that are synchronized on {@link FSDataset}
129 * @param blockId
130 * @return replica from the replicas map
131 */
132 @Deprecated
133 public Replica getReplica(String bpid, long blockId);
134
135 /**
136 * @return replica meta information
137 */
138 public String getReplicaString(String bpid, long blockId);
139
140 /**
141 * @return the generation stamp stored with the block.
142 */
143 public Block getStoredBlock(String bpid, long blkid) throws IOException;
144
145 /**
146 * Returns an input stream at specified offset of the specified block
147 * @param b
148 * @param seekOffset
149 * @return an input stream to read the contents of the specified block,
150 * starting at the offset
151 * @throws IOException
152 */
153 public InputStream getBlockInputStream(ExtendedBlock b, long seekOffset)
154 throws IOException;
155
156 /**
157 * Returns an input stream at specified offset of the specified block
158 * The block is still in the tmp directory and is not finalized
159 * @param b
160 * @param blkoff
161 * @param ckoff
162 * @return an input stream to read the contents of the specified block,
163 * starting at the offset
164 * @throws IOException
165 */
166 public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkoff,
167 long ckoff) throws IOException;
168
169 /**
170 * Creates a temporary replica and returns the meta information of the replica
171 *
172 * @param b block
173 * @return the meta info of the replica which is being written to
174 * @throws IOException if an error occurs
175 */
176 public ReplicaInPipelineInterface createTemporary(ExtendedBlock b
177 ) throws IOException;
178
179 /**
180 * Creates a RBW replica and returns the meta info of the replica
181 *
182 * @param b block
183 * @return the meta info of the replica which is being written to
184 * @throws IOException if an error occurs
185 */
186 public ReplicaInPipelineInterface createRbw(ExtendedBlock b
187 ) throws IOException;
188
189 /**
190 * Recovers a RBW replica and returns the meta info of the replica
191 *
192 * @param b block
193 * @param newGS the new generation stamp for the replica
194 * @param minBytesRcvd the minimum number of bytes that the replica could have
195 * @param maxBytesRcvd the maximum number of bytes that the replica could have
196 * @return the meta info of the replica which is being written to
197 * @throws IOException if an error occurs
198 */
199 public ReplicaInPipelineInterface recoverRbw(ExtendedBlock b,
200 long newGS, long minBytesRcvd, long maxBytesRcvd) throws IOException;
201
202 /**
203 * Covert a temporary replica to a RBW.
204 * @param temporary the temporary replica being converted
205 * @return the result RBW
206 */
207 public ReplicaInPipelineInterface convertTemporaryToRbw(
208 ExtendedBlock temporary) throws IOException;
209
210 /**
211 * Append to a finalized replica and returns the meta info of the replica
212 *
213 * @param b block
214 * @param newGS the new generation stamp for the replica
215 * @param expectedBlockLen the number of bytes the replica is expected to have
216 * @return the meata info of the replica which is being written to
217 * @throws IOException
218 */
219 public ReplicaInPipelineInterface append(ExtendedBlock b, long newGS,
220 long expectedBlockLen) throws IOException;
221
222 /**
223 * Recover a failed append to a finalized replica
224 * and returns the meta info of the replica
225 *
226 * @param b block
227 * @param newGS the new generation stamp for the replica
228 * @param expectedBlockLen the number of bytes the replica is expected to have
229 * @return the meta info of the replica which is being written to
230 * @throws IOException
231 */
232 public ReplicaInPipelineInterface recoverAppend(ExtendedBlock b, long newGS,
233 long expectedBlockLen) throws IOException;
234
235 /**
236 * Recover a failed pipeline close
237 * It bumps the replica's generation stamp and finalize it if RBW replica
238 *
239 * @param b block
240 * @param newGS the new generation stamp for the replica
241 * @param expectedBlockLen the number of bytes the replica is expected to have
242 * @throws IOException
243 */
244 public void recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen
245 ) throws IOException;
246
247 /**
248 * Finalizes the block previously opened for writing using writeToBlock.
249 * The block size is what is in the parameter b and it must match the amount
250 * of data written
251 * @param b
252 * @throws IOException
253 */
254 public void finalizeBlock(ExtendedBlock b) throws IOException;
255
256 /**
257 * Unfinalizes the block previously opened for writing using writeToBlock.
258 * The temporary file associated with this block is deleted.
259 * @param b
260 * @throws IOException
261 */
262 public void unfinalizeBlock(ExtendedBlock b) throws IOException;
263
264 /**
265 * Returns the block report - the full list of blocks stored under a
266 * block pool
267 * @param bpid Block Pool Id
268 * @return - the block report - the full list of blocks stored
269 */
270 public BlockListAsLongs getBlockReport(String bpid);
271
272 /** Does the dataset contain the block? */
273 public boolean contains(ExtendedBlock block);
274
275 /**
276 * Is the block valid?
277 * @param b
278 * @return - true if the specified block is valid
279 */
280 public boolean isValidBlock(ExtendedBlock b);
281
282 /**
283 * Is the block a valid RBW?
284 * @param b
285 * @return - true if the specified block is a valid RBW
286 */
287 public boolean isValidRbw(ExtendedBlock b);
288
289 /**
290 * Invalidates the specified blocks
291 * @param bpid Block pool Id
292 * @param invalidBlks - the blocks to be invalidated
293 * @throws IOException
294 */
295 public void invalidate(String bpid, Block invalidBlks[]) throws IOException;
296
297 /**
298 * Check if all the data directories are healthy
299 * @throws DiskErrorException
300 */
301 public void checkDataDir() throws DiskErrorException;
302
303 /**
304 * Shutdown the FSDataset
305 */
306 public void shutdown();
307
308 /**
309 * Sets the file pointer of the checksum stream so that the last checksum
310 * will be overwritten
311 * @param b block
312 * @param outs The streams for the data file and checksum file
313 * @param checksumSize number of bytes each checksum has
314 * @throws IOException
315 */
316 public void adjustCrcChannelPosition(ExtendedBlock b,
317 ReplicaOutputStreams outs, int checksumSize) throws IOException;
318
319 /**
320 * Checks how many valid storage volumes there are in the DataNode.
321 * @return true if more than the minimum number of valid volumes are left
322 * in the FSDataSet.
323 */
324 public boolean hasEnoughResource();
325
326 /**
327 * Get visible length of the specified replica.
328 */
329 long getReplicaVisibleLength(final ExtendedBlock block) throws IOException;
330
331 /**
332 * Initialize a replica recovery.
333 * @return actual state of the replica on this data-node or
334 * null if data-node does not have the replica.
335 */
336 public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock
337 ) throws IOException;
338
339 /**
340 * Update replica's generation stamp and length and finalize it.
341 * @return the ID of storage that stores the block
342 */
343 public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
344 long recoveryId, long newLength) throws IOException;
345
346 /**
347 * add new block pool ID
348 * @param bpid Block pool Id
349 * @param conf Configuration
350 */
351 public void addBlockPool(String bpid, Configuration conf) throws IOException;
352
353 /**
354 * Shutdown and remove the block pool from underlying storage.
355 * @param bpid Block pool Id to be removed
356 */
357 public void shutdownBlockPool(String bpid) ;
358
359 /**
360 * Deletes the block pool directories. If force is false, directories are
361 * deleted only if no block files exist for the block pool. If force
362 * is true entire directory for the blockpool is deleted along with its
363 * contents.
364 * @param bpid BlockPool Id to be deleted.
365 * @param force If force is false, directories are deleted only if no
366 * block files exist for the block pool, otherwise entire
367 * directory for the blockpool is deleted along with its contents.
368 * @throws IOException
369 */
370 public void deleteBlockPool(String bpid, boolean force) throws IOException;
371
372 /**
373 * Get {@link BlockLocalPathInfo} for the given block.
374 */
375 public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock b
376 ) throws IOException;
377
378 /**
379 * Get a {@link HdfsBlocksMetadata} corresponding to the list of blocks in
380 * <code>blocks</code>.
381 *
382 * @param blocks List of blocks for which to return metadata
383 * @return metadata Metadata for the list of blocks
384 * @throws IOException
385 */
386 public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks)
387 throws IOException;
388 }