001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode.metrics;
019
020import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
021import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
022
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.hdfs.DFSConfigKeys;
025import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
026import org.apache.hadoop.metrics2.MetricsSystem;
027import org.apache.hadoop.metrics2.annotation.Metric;
028import org.apache.hadoop.metrics2.annotation.Metrics;
029import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
030import org.apache.hadoop.metrics2.lib.MetricsRegistry;
031import org.apache.hadoop.metrics2.lib.MutableCounterLong;
032import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
033import org.apache.hadoop.metrics2.lib.MutableQuantiles;
034import org.apache.hadoop.metrics2.lib.MutableRate;
035import org.apache.hadoop.metrics2.source.JvmMetrics;
036
037/**
038 * This class is for maintaining  the various NameNode activity statistics
039 * and publishing them through the metrics interfaces.
040 */
041@Metrics(name="NameNodeActivity", about="NameNode metrics", context="dfs")
042public class NameNodeMetrics {
043  final MetricsRegistry registry = new MetricsRegistry("namenode");
044
045  @Metric MutableCounterLong createFileOps;
046  @Metric MutableCounterLong filesCreated;
047  @Metric MutableCounterLong filesAppended;
048  @Metric MutableCounterLong getBlockLocations;
049  @Metric MutableCounterLong filesRenamed;
050  @Metric MutableCounterLong filesTruncated;
051  @Metric MutableCounterLong getListingOps;
052  @Metric MutableCounterLong deleteFileOps;
053  @Metric("Number of files/dirs deleted by delete or rename operations")
054  MutableCounterLong filesDeleted;
055  @Metric MutableCounterLong fileInfoOps;
056  @Metric MutableCounterLong addBlockOps;
057  @Metric MutableCounterLong getAdditionalDatanodeOps;
058  @Metric MutableCounterLong createSymlinkOps;
059  @Metric MutableCounterLong getLinkTargetOps;
060  @Metric MutableCounterLong filesInGetListingOps;
061  @Metric("Number of allowSnapshot operations")
062  MutableCounterLong allowSnapshotOps;
063  @Metric("Number of disallowSnapshot operations")
064  MutableCounterLong disallowSnapshotOps;
065  @Metric("Number of createSnapshot operations")
066  MutableCounterLong createSnapshotOps;
067  @Metric("Number of deleteSnapshot operations")
068  MutableCounterLong deleteSnapshotOps;
069  @Metric("Number of renameSnapshot operations")
070  MutableCounterLong renameSnapshotOps;
071  @Metric("Number of listSnapshottableDirectory operations")
072  MutableCounterLong listSnapshottableDirOps;
073  @Metric("Number of snapshotDiffReport operations")
074  MutableCounterLong snapshotDiffReportOps;
075  @Metric("Number of blockReceivedAndDeleted calls")
076  MutableCounterLong blockReceivedAndDeletedOps;
077  @Metric("Number of blockReports from individual storages")
078  MutableCounterLong storageBlockReportOps;
079  @Metric("Number of blockReports and blockReceivedAndDeleted queued")
080  MutableGaugeInt blockOpsQueued;
081  @Metric("Number of blockReports and blockReceivedAndDeleted batch processed")
082  MutableCounterLong blockOpsBatched;
083
084  @Metric("Number of file system operations")
085  public long totalFileOps(){
086    return
087      getBlockLocations.value() +
088      createFileOps.value() +
089      filesAppended.value() +
090      addBlockOps.value() +
091      getAdditionalDatanodeOps.value() +
092      filesRenamed.value() +
093      filesTruncated.value() +
094      deleteFileOps.value() +
095      getListingOps.value() +
096      fileInfoOps.value() +
097      getLinkTargetOps.value() +
098      createSnapshotOps.value() +
099      deleteSnapshotOps.value() +
100      allowSnapshotOps.value() +
101      disallowSnapshotOps.value() +
102      renameSnapshotOps.value() +
103      listSnapshottableDirOps.value() +
104      createSymlinkOps.value() +
105      snapshotDiffReportOps.value();
106  }
107
108
109  @Metric("Journal transactions") MutableRate transactions;
110  @Metric("Journal syncs") MutableRate syncs;
111  final MutableQuantiles[] syncsQuantiles;
112  @Metric("Journal transactions batched in sync")
113  MutableCounterLong transactionsBatchedInSync;
114  @Metric("Block report") MutableRate blockReport;
115  final MutableQuantiles[] blockReportQuantiles;
116  @Metric("Cache report") MutableRate cacheReport;
117  final MutableQuantiles[] cacheReportQuantiles;
118  @Metric("Generate EDEK time") private MutableRate generateEDEKTime;
119  private final MutableQuantiles[] generateEDEKTimeQuantiles;
120  @Metric("Warm-up EDEK time") private MutableRate warmUpEDEKTime;
121  private final MutableQuantiles[] warmUpEDEKTimeQuantiles;
122
123  @Metric("Duration in SafeMode at startup in msec")
124  MutableGaugeInt safeModeTime;
125  @Metric("Time loading FS Image at startup in msec")
126  MutableGaugeInt fsImageLoadTime;
127
128  @Metric("GetImageServlet getEdit")
129  MutableRate getEdit;
130  @Metric("GetImageServlet getImage")
131  MutableRate getImage;
132  @Metric("GetImageServlet putImage")
133  MutableRate putImage;
134
135  JvmMetrics jvmMetrics = null;
136  
137  NameNodeMetrics(String processName, String sessionId, int[] intervals,
138      final JvmMetrics jvmMetrics) {
139    this.jvmMetrics = jvmMetrics;
140    registry.tag(ProcessName, processName).tag(SessionId, sessionId);
141    
142    final int len = intervals.length;
143    syncsQuantiles = new MutableQuantiles[len];
144    blockReportQuantiles = new MutableQuantiles[len];
145    cacheReportQuantiles = new MutableQuantiles[len];
146    generateEDEKTimeQuantiles = new MutableQuantiles[len];
147    warmUpEDEKTimeQuantiles = new MutableQuantiles[len];
148    
149    for (int i = 0; i < len; i++) {
150      int interval = intervals[i];
151      syncsQuantiles[i] = registry.newQuantiles(
152          "syncs" + interval + "s",
153          "Journal syncs", "ops", "latency", interval);
154      blockReportQuantiles[i] = registry.newQuantiles(
155          "blockReport" + interval + "s", 
156          "Block report", "ops", "latency", interval);
157      cacheReportQuantiles[i] = registry.newQuantiles(
158          "cacheReport" + interval + "s",
159          "Cache report", "ops", "latency", interval);
160      generateEDEKTimeQuantiles[i] = registry.newQuantiles(
161          "generateEDEKTime" + interval + "s",
162          "Generate EDEK time", "ops", "latency", interval);
163      warmUpEDEKTimeQuantiles[i] = registry.newQuantiles(
164          "warmupEDEKTime" + interval + "s",
165          "Warm up EDEK time", "ops", "latency", interval);
166    }
167  }
168
169  public static NameNodeMetrics create(Configuration conf, NamenodeRole r) {
170    String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
171    String processName = r.toString();
172    MetricsSystem ms = DefaultMetricsSystem.instance();
173    JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
174    
175    // Percentile measurement is off by default, by watching no intervals
176    int[] intervals = 
177        conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
178    return ms.register(new NameNodeMetrics(processName, sessionId,
179        intervals, jm));
180  }
181
182  public JvmMetrics getJvmMetrics() {
183    return jvmMetrics;
184  }
185  
186  public void shutdown() {
187    DefaultMetricsSystem.shutdown();
188  }
189
190  public void incrGetBlockLocations() {
191    getBlockLocations.incr();
192  }
193
194  public void incrFilesCreated() {
195    filesCreated.incr();
196  }
197
198  public void incrCreateFileOps() {
199    createFileOps.incr();
200  }
201
202  public void incrFilesAppended() {
203    filesAppended.incr();
204  }
205
206  public void incrAddBlockOps() {
207    addBlockOps.incr();
208  }
209  
210  public void incrGetAdditionalDatanodeOps() {
211    getAdditionalDatanodeOps.incr();
212  }
213
214  public void incrFilesRenamed() {
215    filesRenamed.incr();
216  }
217
218  public void incrFilesTruncated() {
219    filesTruncated.incr();
220  }
221
222  public void incrFilesDeleted(long delta) {
223    filesDeleted.incr(delta);
224  }
225
226  public void incrDeleteFileOps() {
227    deleteFileOps.incr();
228  }
229
230  public void incrGetListingOps() {
231    getListingOps.incr();
232  }
233
234  public void incrFilesInGetListingOps(int delta) {
235    filesInGetListingOps.incr(delta);
236  }
237
238  public void incrFileInfoOps() {
239    fileInfoOps.incr();
240  }
241
242  public void incrCreateSymlinkOps() {
243    createSymlinkOps.incr();
244  }
245
246  public void incrGetLinkTargetOps() {
247    getLinkTargetOps.incr();
248  }
249
250  public void incrAllowSnapshotOps() {
251    allowSnapshotOps.incr();
252  }
253  
254  public void incrDisAllowSnapshotOps() {
255    disallowSnapshotOps.incr();
256  }
257  
258  public void incrCreateSnapshotOps() {
259    createSnapshotOps.incr();
260  }
261  
262  public void incrDeleteSnapshotOps() {
263    deleteSnapshotOps.incr();
264  }
265  
266  public void incrRenameSnapshotOps() {
267    renameSnapshotOps.incr();
268  }
269  
270  public void incrListSnapshottableDirOps() {
271    listSnapshottableDirOps.incr();
272  }
273  
274  public void incrSnapshotDiffReportOps() {
275    snapshotDiffReportOps.incr();
276  }
277  
278  public void incrBlockReceivedAndDeletedOps() {
279    blockReceivedAndDeletedOps.incr();
280  }
281  
282  public void incrStorageBlockReportOps() {
283    storageBlockReportOps.incr();
284  }
285
286  public void setBlockOpsQueued(int size) {
287    blockOpsQueued.set(size);
288  }
289
290  public void addBlockOpsBatched(int count) {
291    blockOpsBatched.incr(count);
292  }
293
294  public void addTransaction(long latency) {
295    transactions.add(latency);
296  }
297
298  public void incrTransactionsBatchedInSync(long count) {
299    transactionsBatchedInSync.incr(count);
300  }
301
302  public void addSync(long elapsed) {
303    syncs.add(elapsed);
304    for (MutableQuantiles q : syncsQuantiles) {
305      q.add(elapsed);
306    }
307  }
308
309  public void setFsImageLoadTime(long elapsed) {
310    fsImageLoadTime.set((int) elapsed);
311  }
312
313  public void addBlockReport(long latency) {
314    blockReport.add(latency);
315    for (MutableQuantiles q : blockReportQuantiles) {
316      q.add(latency);
317    }
318  }
319
320  public void addCacheBlockReport(long latency) {
321    cacheReport.add(latency);
322    for (MutableQuantiles q : cacheReportQuantiles) {
323      q.add(latency);
324    }
325  }
326
327  public void setSafeModeTime(long elapsed) {
328    safeModeTime.set((int) elapsed);
329  }
330
331  public void addGetEdit(long latency) {
332    getEdit.add(latency);
333  }
334
335  public void addGetImage(long latency) {
336    getImage.add(latency);
337  }
338
339  public void addPutImage(long latency) {
340    putImage.add(latency);
341  }
342
343  public void addGenerateEDEKTime(long latency) {
344    generateEDEKTime.add(latency);
345    for (MutableQuantiles q : generateEDEKTimeQuantiles) {
346      q.add(latency);
347    }
348  }
349
350  public void addWarmUpEDEKTime(long latency) {
351    warmUpEDEKTime.add(latency);
352    for (MutableQuantiles q : warmUpEDEKTimeQuantiles) {
353      q.add(latency);
354    }
355  }
356}