001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.datanode;
019
020 import java.io.File;
021 import java.io.FileInputStream;
022 import java.io.FileOutputStream;
023 import java.io.IOException;
024 import java.util.ArrayList;
025 import java.util.HashMap;
026 import java.util.List;
027 import java.util.Map;
028
029 import org.apache.hadoop.classification.InterfaceAudience;
030 import org.apache.hadoop.fs.FileUtil;
031 import org.apache.hadoop.fs.HardLink;
032 import org.apache.hadoop.hdfs.protocol.Block;
033 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
034 import org.apache.hadoop.io.IOUtils;
035
036 import com.google.common.annotations.VisibleForTesting;
037
038 /**
039 * This class is used by datanodes to maintain meta data of its replicas.
040 * It provides a general interface for meta information of a replica.
041 */
042 @InterfaceAudience.Private
043 abstract public class ReplicaInfo extends Block implements Replica {
044
045 /** volume where the replica belongs */
046 private FsVolumeSpi volume;
047
048 /** directory where block & meta files belong */
049
050 /**
051 * Base directory containing numerically-identified sub directories and
052 * possibly blocks.
053 */
054 private File baseDir;
055
056 /**
057 * Ints representing the sub directory path from base dir to the directory
058 * containing this replica.
059 */
060 private int[] subDirs;
061
062 private static final Map<String, File> internedBaseDirs = new HashMap<String, File>();
063
064 /**
065 * Constructor for a zero length replica
066 * @param blockId block id
067 * @param genStamp replica generation stamp
068 * @param vol volume where replica is located
069 * @param dir directory path where block and meta files are located
070 */
071 ReplicaInfo(long blockId, long genStamp, FsVolumeSpi vol, File dir) {
072 this( blockId, 0L, genStamp, vol, dir);
073 }
074
075 /**
076 * Constructor
077 * @param block a block
078 * @param vol volume where replica is located
079 * @param dir directory path where block and meta files are located
080 */
081 ReplicaInfo(Block block, FsVolumeSpi vol, File dir) {
082 this(block.getBlockId(), block.getNumBytes(),
083 block.getGenerationStamp(), vol, dir);
084 }
085
086 /**
087 * Constructor
088 * @param blockId block id
089 * @param len replica length
090 * @param genStamp replica generation stamp
091 * @param vol volume where replica is located
092 * @param dir directory path where block and meta files are located
093 */
094 ReplicaInfo(long blockId, long len, long genStamp,
095 FsVolumeSpi vol, File dir) {
096 super(blockId, len, genStamp);
097 this.volume = vol;
098 setDirInternal(dir);
099 }
100
101 /**
102 * Copy constructor.
103 * @param from
104 */
105 ReplicaInfo(ReplicaInfo from) {
106 this(from, from.getVolume(), from.getDir());
107 }
108
109 /**
110 * Get the full path of this replica's data file
111 * @return the full path of this replica's data file
112 */
113 public File getBlockFile() {
114 return new File(getDir(), getBlockName());
115 }
116
117 /**
118 * Get the full path of this replica's meta file
119 * @return the full path of this replica's meta file
120 */
121 public File getMetaFile() {
122 return new File(getDir(),
123 DatanodeUtil.getMetaName(getBlockName(), getGenerationStamp()));
124 }
125
126 /**
127 * Get the volume where this replica is located on disk
128 * @return the volume where this replica is located on disk
129 */
130 public FsVolumeSpi getVolume() {
131 return volume;
132 }
133
134 /**
135 * Set the volume where this replica is located on disk
136 */
137 void setVolume(FsVolumeSpi vol) {
138 this.volume = vol;
139 }
140
141 /**
142 * Get the storageUuid of the volume that stores this replica.
143 */
144 @Override
145 public String getStorageUuid() {
146 return volume.getStorageID();
147 }
148
149 /**
150 * Return the parent directory path where this replica is located
151 * @return the parent directory path where this replica is located
152 */
153 File getDir() {
154 if (subDirs == null) {
155 return null;
156 }
157
158 StringBuilder sb = new StringBuilder();
159 for (int i : subDirs) {
160 sb.append(DataStorage.BLOCK_SUBDIR_PREFIX);
161 sb.append(i);
162 sb.append("/");
163 }
164 File ret = new File(baseDir, sb.toString());
165 return ret;
166 }
167
168 /**
169 * Set the parent directory where this replica is located
170 * @param dir the parent directory where the replica is located
171 */
172 public void setDir(File dir) {
173 setDirInternal(dir);
174 }
175
176 private void setDirInternal(File dir) {
177 if (dir == null) {
178 subDirs = null;
179 baseDir = null;
180 return;
181 }
182
183 ReplicaDirInfo replicaDirInfo = parseSubDirs(dir);
184 this.subDirs = replicaDirInfo.subDirs;
185
186 synchronized (internedBaseDirs) {
187 if (!internedBaseDirs.containsKey(replicaDirInfo.baseDirPath)) {
188 // Create a new String path of this file and make a brand new File object
189 // to guarantee we drop the reference to the underlying char[] storage.
190 File baseDir = new File(new String(replicaDirInfo.baseDirPath));
191 internedBaseDirs.put(replicaDirInfo.baseDirPath, baseDir);
192 }
193 this.baseDir = internedBaseDirs.get(replicaDirInfo.baseDirPath);
194 }
195 }
196
197 @VisibleForTesting
198 public static class ReplicaDirInfo {
199 @VisibleForTesting
200 public String baseDirPath;
201
202 @VisibleForTesting
203 public int[] subDirs;
204 }
205
206 @VisibleForTesting
207 public static ReplicaDirInfo parseSubDirs(File dir) {
208 ReplicaDirInfo ret = new ReplicaDirInfo();
209
210 File currentDir = dir;
211 List<Integer> subDirList = new ArrayList<Integer>();
212 while (currentDir.getName().startsWith(DataStorage.BLOCK_SUBDIR_PREFIX)) {
213 // Prepend the integer into the list.
214 subDirList.add(0, Integer.parseInt(currentDir.getName().replaceFirst(
215 DataStorage.BLOCK_SUBDIR_PREFIX, "")));
216 currentDir = currentDir.getParentFile();
217 }
218 ret.subDirs = new int[subDirList.size()];
219 for (int i = 0; i < subDirList.size(); i++) {
220 ret.subDirs[i] = subDirList.get(i);
221 }
222
223 ret.baseDirPath = currentDir.getAbsolutePath();
224
225 return ret;
226 }
227
228 /**
229 * check if this replica has already been unlinked.
230 * @return true if the replica has already been unlinked
231 * or no need to be detached; false otherwise
232 */
233 public boolean isUnlinked() {
234 return true; // no need to be unlinked
235 }
236
237 /**
238 * set that this replica is unlinked
239 */
240 public void setUnlinked() {
241 // no need to be unlinked
242 }
243
244 /**
245 * Copy specified file into a temporary file. Then rename the
246 * temporary file to the original name. This will cause any
247 * hardlinks to the original file to be removed. The temporary
248 * files are created in the same directory. The temporary files will
249 * be recovered (especially on Windows) on datanode restart.
250 */
251 private void unlinkFile(File file, Block b) throws IOException {
252 File tmpFile = DatanodeUtil.createTmpFile(b, DatanodeUtil.getUnlinkTmpFile(file));
253 try {
254 FileInputStream in = new FileInputStream(file);
255 try {
256 FileOutputStream out = new FileOutputStream(tmpFile);
257 try {
258 IOUtils.copyBytes(in, out, 16*1024);
259 } finally {
260 out.close();
261 }
262 } finally {
263 in.close();
264 }
265 if (file.length() != tmpFile.length()) {
266 throw new IOException("Copy of file " + file + " size " + file.length()+
267 " into file " + tmpFile +
268 " resulted in a size of " + tmpFile.length());
269 }
270 FileUtil.replaceFile(tmpFile, file);
271 } catch (IOException e) {
272 boolean done = tmpFile.delete();
273 if (!done) {
274 DataNode.LOG.info("detachFile failed to delete temporary file " +
275 tmpFile);
276 }
277 throw e;
278 }
279 }
280
281 /**
282 * Remove a hard link by copying the block to a temporary place and
283 * then moving it back
284 * @param numLinks number of hard links
285 * @return true if copy is successful;
286 * false if it is already detached or no need to be detached
287 * @throws IOException if there is any copy error
288 */
289 public boolean unlinkBlock(int numLinks) throws IOException {
290 if (isUnlinked()) {
291 return false;
292 }
293 File file = getBlockFile();
294 if (file == null || getVolume() == null) {
295 throw new IOException("detachBlock:Block not found. " + this);
296 }
297 File meta = getMetaFile();
298
299 if (HardLink.getLinkCount(file) > numLinks) {
300 DataNode.LOG.info("CopyOnWrite for block " + this);
301 unlinkFile(file, this);
302 }
303 if (HardLink.getLinkCount(meta) > numLinks) {
304 unlinkFile(meta, this);
305 }
306 setUnlinked();
307 return true;
308 }
309
310 /**
311 * Set this replica's generation stamp to be a newer one
312 * @param newGS new generation stamp
313 * @throws IOException is the new generation stamp is not greater than the current one
314 */
315 void setNewerGenerationStamp(long newGS) throws IOException {
316 long curGS = getGenerationStamp();
317 if (newGS <= curGS) {
318 throw new IOException("New generation stamp (" + newGS
319 + ") must be greater than current one (" + curGS + ")");
320 }
321 setGenerationStamp(newGS);
322 }
323
324 @Override //Object
325 public String toString() {
326 return getClass().getSimpleName()
327 + ", " + super.toString()
328 + ", " + getState()
329 + "\n getNumBytes() = " + getNumBytes()
330 + "\n getBytesOnDisk() = " + getBytesOnDisk()
331 + "\n getVisibleLength()= " + getVisibleLength()
332 + "\n getVolume() = " + getVolume()
333 + "\n getBlockFile() = " + getBlockFile();
334 }
335 }