001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.security.token.block;
020
021 import java.io.ByteArrayInputStream;
022 import java.io.DataInputStream;
023 import java.io.IOException;
024 import java.security.SecureRandom;
025 import java.util.Arrays;
026 import java.util.EnumSet;
027 import java.util.HashMap;
028 import java.util.Iterator;
029 import java.util.Map;
030
031 import org.apache.commons.logging.Log;
032 import org.apache.commons.logging.LogFactory;
033 import org.apache.hadoop.classification.InterfaceAudience;
034 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
035 import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException;
036 import org.apache.hadoop.io.WritableUtils;
037 import org.apache.hadoop.security.UserGroupInformation;
038 import org.apache.hadoop.security.token.SecretManager;
039 import org.apache.hadoop.security.token.Token;
040 import org.apache.hadoop.util.Time;
041
042 import com.google.common.annotations.VisibleForTesting;
043 import com.google.common.base.Preconditions;
044
045 /**
046 * BlockTokenSecretManager can be instantiated in 2 modes, master mode and slave
047 * mode. Master can generate new block keys and export block keys to slaves,
048 * while slaves can only import and use block keys received from master. Both
049 * master and slave can generate and verify block tokens. Typically, master mode
050 * is used by NN and slave mode is used by DN.
051 */
052 @InterfaceAudience.Private
053 public class BlockTokenSecretManager extends
054 SecretManager<BlockTokenIdentifier> {
055 public static final Log LOG = LogFactory
056 .getLog(BlockTokenSecretManager.class);
057
058 // We use these in an HA setup to ensure that the pair of NNs produce block
059 // token serial numbers that are in different ranges.
060 private static final int LOW_MASK = ~(1 << 31);
061
062 public static final Token<BlockTokenIdentifier> DUMMY_TOKEN = new Token<BlockTokenIdentifier>();
063
064 private final boolean isMaster;
065 private int nnIndex;
066
067 /**
068 * keyUpdateInterval is the interval that NN updates its block keys. It should
069 * be set long enough so that all live DN's and Balancer should have sync'ed
070 * their block keys with NN at least once during each interval.
071 */
072 private long keyUpdateInterval;
073 private volatile long tokenLifetime;
074 private int serialNo;
075 private BlockKey currentKey;
076 private BlockKey nextKey;
077 private final Map<Integer, BlockKey> allKeys;
078 private String blockPoolId;
079 private final String encryptionAlgorithm;
080
081 private final SecureRandom nonceGenerator = new SecureRandom();
082
083 public static enum AccessMode {
084 READ, WRITE, COPY, REPLACE
085 };
086
087 /**
088 * Constructor for slaves.
089 *
090 * @param keyUpdateInterval how often a new key will be generated
091 * @param tokenLifetime how long an individual token is valid
092 */
093 public BlockTokenSecretManager(long keyUpdateInterval,
094 long tokenLifetime, String blockPoolId, String encryptionAlgorithm) {
095 this(false, keyUpdateInterval, tokenLifetime, blockPoolId,
096 encryptionAlgorithm);
097 }
098
099 /**
100 * Constructor for masters.
101 *
102 * @param keyUpdateInterval how often a new key will be generated
103 * @param tokenLifetime how long an individual token is valid
104 * @param nnIndex namenode index
105 * @param blockPoolId block pool ID
106 * @param encryptionAlgorithm encryption algorithm to use
107 */
108 public BlockTokenSecretManager(long keyUpdateInterval,
109 long tokenLifetime, int nnIndex, String blockPoolId,
110 String encryptionAlgorithm) {
111 this(true, keyUpdateInterval, tokenLifetime, blockPoolId,
112 encryptionAlgorithm);
113 Preconditions.checkArgument(nnIndex == 0 || nnIndex == 1);
114 this.nnIndex = nnIndex;
115 setSerialNo(new SecureRandom().nextInt());
116 generateKeys();
117 }
118
119 private BlockTokenSecretManager(boolean isMaster, long keyUpdateInterval,
120 long tokenLifetime, String blockPoolId, String encryptionAlgorithm) {
121 this.isMaster = isMaster;
122 this.keyUpdateInterval = keyUpdateInterval;
123 this.tokenLifetime = tokenLifetime;
124 this.allKeys = new HashMap<Integer, BlockKey>();
125 this.blockPoolId = blockPoolId;
126 this.encryptionAlgorithm = encryptionAlgorithm;
127 generateKeys();
128 }
129
130 @VisibleForTesting
131 public synchronized void setSerialNo(int serialNo) {
132 this.serialNo = (serialNo & LOW_MASK) | (nnIndex << 31);
133 }
134
135 public void setBlockPoolId(String blockPoolId) {
136 this.blockPoolId = blockPoolId;
137 }
138
139 /** Initialize block keys */
140 private synchronized void generateKeys() {
141 if (!isMaster)
142 return;
143 /*
144 * Need to set estimated expiry dates for currentKey and nextKey so that if
145 * NN crashes, DN can still expire those keys. NN will stop using the newly
146 * generated currentKey after the first keyUpdateInterval, however it may
147 * still be used by DN and Balancer to generate new tokens before they get a
148 * chance to sync their keys with NN. Since we require keyUpdInterval to be
149 * long enough so that all live DN's and Balancer will sync their keys with
150 * NN at least once during the period, the estimated expiry date for
151 * currentKey is set to now() + 2 * keyUpdateInterval + tokenLifetime.
152 * Similarly, the estimated expiry date for nextKey is one keyUpdateInterval
153 * more.
154 */
155 setSerialNo(serialNo + 1);
156 currentKey = new BlockKey(serialNo, Time.now() + 2
157 * keyUpdateInterval + tokenLifetime, generateSecret());
158 setSerialNo(serialNo + 1);
159 nextKey = new BlockKey(serialNo, Time.now() + 3
160 * keyUpdateInterval + tokenLifetime, generateSecret());
161 allKeys.put(currentKey.getKeyId(), currentKey);
162 allKeys.put(nextKey.getKeyId(), nextKey);
163 }
164
165 /** Export block keys, only to be used in master mode */
166 public synchronized ExportedBlockKeys exportKeys() {
167 if (!isMaster)
168 return null;
169 if (LOG.isDebugEnabled())
170 LOG.debug("Exporting access keys");
171 return new ExportedBlockKeys(true, keyUpdateInterval, tokenLifetime,
172 currentKey, allKeys.values().toArray(new BlockKey[0]));
173 }
174
175 private synchronized void removeExpiredKeys() {
176 long now = Time.now();
177 for (Iterator<Map.Entry<Integer, BlockKey>> it = allKeys.entrySet()
178 .iterator(); it.hasNext();) {
179 Map.Entry<Integer, BlockKey> e = it.next();
180 if (e.getValue().getExpiryDate() < now) {
181 it.remove();
182 }
183 }
184 }
185
186 /**
187 * Set block keys, only to be used in slave mode
188 */
189 public synchronized void addKeys(ExportedBlockKeys exportedKeys)
190 throws IOException {
191 if (isMaster || exportedKeys == null)
192 return;
193 LOG.info("Setting block keys");
194 removeExpiredKeys();
195 this.currentKey = exportedKeys.getCurrentKey();
196 BlockKey[] receivedKeys = exportedKeys.getAllKeys();
197 for (int i = 0; i < receivedKeys.length; i++) {
198 if (receivedKeys[i] == null)
199 continue;
200 this.allKeys.put(receivedKeys[i].getKeyId(), receivedKeys[i]);
201 }
202 }
203
204 /**
205 * Update block keys if update time > update interval.
206 * @return true if the keys are updated.
207 */
208 public synchronized boolean updateKeys(final long updateTime) throws IOException {
209 if (updateTime > keyUpdateInterval) {
210 return updateKeys();
211 }
212 return false;
213 }
214
215 /**
216 * Update block keys, only to be used in master mode
217 */
218 synchronized boolean updateKeys() throws IOException {
219 if (!isMaster)
220 return false;
221
222 LOG.info("Updating block keys");
223 removeExpiredKeys();
224 // set final expiry date of retiring currentKey
225 allKeys.put(currentKey.getKeyId(), new BlockKey(currentKey.getKeyId(),
226 Time.now() + keyUpdateInterval + tokenLifetime,
227 currentKey.getKey()));
228 // update the estimated expiry date of new currentKey
229 currentKey = new BlockKey(nextKey.getKeyId(), Time.now()
230 + 2 * keyUpdateInterval + tokenLifetime, nextKey.getKey());
231 allKeys.put(currentKey.getKeyId(), currentKey);
232 // generate a new nextKey
233 setSerialNo(serialNo + 1);
234 nextKey = new BlockKey(serialNo, Time.now() + 3
235 * keyUpdateInterval + tokenLifetime, generateSecret());
236 allKeys.put(nextKey.getKeyId(), nextKey);
237 return true;
238 }
239
240 /** Generate an block token for current user */
241 public Token<BlockTokenIdentifier> generateToken(ExtendedBlock block,
242 EnumSet<AccessMode> modes) throws IOException {
243 UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
244 String userID = (ugi == null ? null : ugi.getShortUserName());
245 return generateToken(userID, block, modes);
246 }
247
248 /** Generate a block token for a specified user */
249 public Token<BlockTokenIdentifier> generateToken(String userId,
250 ExtendedBlock block, EnumSet<AccessMode> modes) throws IOException {
251 BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block
252 .getBlockPoolId(), block.getBlockId(), modes);
253 return new Token<BlockTokenIdentifier>(id, this);
254 }
255
256 /**
257 * Check if access should be allowed. userID is not checked if null. This
258 * method doesn't check if token password is correct. It should be used only
259 * when token password has already been verified (e.g., in the RPC layer).
260 */
261 public void checkAccess(BlockTokenIdentifier id, String userId,
262 ExtendedBlock block, AccessMode mode) throws InvalidToken {
263 if (LOG.isDebugEnabled()) {
264 LOG.debug("Checking access for user=" + userId + ", block=" + block
265 + ", access mode=" + mode + " using " + id.toString());
266 }
267 if (userId != null && !userId.equals(id.getUserId())) {
268 throw new InvalidToken("Block token with " + id.toString()
269 + " doesn't belong to user " + userId);
270 }
271 if (!id.getBlockPoolId().equals(block.getBlockPoolId())) {
272 throw new InvalidToken("Block token with " + id.toString()
273 + " doesn't apply to block " + block);
274 }
275 if (id.getBlockId() != block.getBlockId()) {
276 throw new InvalidToken("Block token with " + id.toString()
277 + " doesn't apply to block " + block);
278 }
279 if (isExpired(id.getExpiryDate())) {
280 throw new InvalidToken("Block token with " + id.toString()
281 + " is expired.");
282 }
283 if (!id.getAccessModes().contains(mode)) {
284 throw new InvalidToken("Block token with " + id.toString()
285 + " doesn't have " + mode + " permission");
286 }
287 }
288
289 /** Check if access should be allowed. userID is not checked if null */
290 public void checkAccess(Token<BlockTokenIdentifier> token, String userId,
291 ExtendedBlock block, AccessMode mode) throws InvalidToken {
292 BlockTokenIdentifier id = new BlockTokenIdentifier();
293 try {
294 id.readFields(new DataInputStream(new ByteArrayInputStream(token
295 .getIdentifier())));
296 } catch (IOException e) {
297 throw new InvalidToken(
298 "Unable to de-serialize block token identifier for user=" + userId
299 + ", block=" + block + ", access mode=" + mode);
300 }
301 checkAccess(id, userId, block, mode);
302 if (!Arrays.equals(retrievePassword(id), token.getPassword())) {
303 throw new InvalidToken("Block token with " + id.toString()
304 + " doesn't have the correct token password");
305 }
306 }
307
308 private static boolean isExpired(long expiryDate) {
309 return Time.now() > expiryDate;
310 }
311
312 /**
313 * check if a token is expired. for unit test only. return true when token is
314 * expired, false otherwise
315 */
316 static boolean isTokenExpired(Token<BlockTokenIdentifier> token)
317 throws IOException {
318 ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
319 DataInputStream in = new DataInputStream(buf);
320 long expiryDate = WritableUtils.readVLong(in);
321 return isExpired(expiryDate);
322 }
323
324 /** set token lifetime. */
325 public void setTokenLifetime(long tokenLifetime) {
326 this.tokenLifetime = tokenLifetime;
327 }
328
329 /**
330 * Create an empty block token identifier
331 *
332 * @return a newly created empty block token identifier
333 */
334 @Override
335 public BlockTokenIdentifier createIdentifier() {
336 return new BlockTokenIdentifier();
337 }
338
339 /**
340 * Create a new password/secret for the given block token identifier.
341 *
342 * @param identifier
343 * the block token identifier
344 * @return token password/secret
345 */
346 @Override
347 protected byte[] createPassword(BlockTokenIdentifier identifier) {
348 BlockKey key = null;
349 synchronized (this) {
350 key = currentKey;
351 }
352 if (key == null)
353 throw new IllegalStateException("currentKey hasn't been initialized.");
354 identifier.setExpiryDate(Time.now() + tokenLifetime);
355 identifier.setKeyId(key.getKeyId());
356 if (LOG.isDebugEnabled()) {
357 LOG.debug("Generating block token for " + identifier.toString());
358 }
359 return createPassword(identifier.getBytes(), key.getKey());
360 }
361
362 /**
363 * Look up the token password/secret for the given block token identifier.
364 *
365 * @param identifier
366 * the block token identifier to look up
367 * @return token password/secret as byte[]
368 * @throws InvalidToken
369 */
370 @Override
371 public byte[] retrievePassword(BlockTokenIdentifier identifier)
372 throws InvalidToken {
373 if (isExpired(identifier.getExpiryDate())) {
374 throw new InvalidToken("Block token with " + identifier.toString()
375 + " is expired.");
376 }
377 BlockKey key = null;
378 synchronized (this) {
379 key = allKeys.get(identifier.getKeyId());
380 }
381 if (key == null) {
382 throw new InvalidToken("Can't re-compute password for "
383 + identifier.toString() + ", since the required block key (keyID="
384 + identifier.getKeyId() + ") doesn't exist.");
385 }
386 return createPassword(identifier.getBytes(), key.getKey());
387 }
388
389 /**
390 * Generate a data encryption key for this block pool, using the current
391 * BlockKey.
392 *
393 * @return a data encryption key which may be used to encrypt traffic
394 * over the DataTransferProtocol
395 */
396 public DataEncryptionKey generateDataEncryptionKey() {
397 byte[] nonce = new byte[8];
398 nonceGenerator.nextBytes(nonce);
399 BlockKey key = null;
400 synchronized (this) {
401 key = currentKey;
402 }
403 byte[] encryptionKey = createPassword(nonce, key.getKey());
404 return new DataEncryptionKey(key.getKeyId(), blockPoolId, nonce,
405 encryptionKey, Time.now() + tokenLifetime,
406 encryptionAlgorithm);
407 }
408
409 /**
410 * Recreate an encryption key based on the given key id and nonce.
411 *
412 * @param keyId identifier of the secret key used to generate the encryption key.
413 * @param nonce random value used to create the encryption key
414 * @return the encryption key which corresponds to this (keyId, blockPoolId, nonce)
415 * @throws InvalidEncryptionKeyException
416 */
417 public byte[] retrieveDataEncryptionKey(int keyId, byte[] nonce)
418 throws InvalidEncryptionKeyException {
419 BlockKey key = null;
420 synchronized (this) {
421 key = allKeys.get(keyId);
422 if (key == null) {
423 throw new InvalidEncryptionKeyException("Can't re-compute encryption key"
424 + " for nonce, since the required block key (keyID=" + keyId
425 + ") doesn't exist. Current key: " + currentKey.getKeyId());
426 }
427 }
428 return createPassword(nonce, key.getKey());
429 }
430
431 @VisibleForTesting
432 public synchronized void setKeyUpdateIntervalForTesting(long millis) {
433 this.keyUpdateInterval = millis;
434 }
435
436 @VisibleForTesting
437 public void clearAllKeysForTesting() {
438 allKeys.clear();
439 }
440
441 @VisibleForTesting
442 public synchronized int getSerialNoForTesting() {
443 return serialNo;
444 }
445
446 }