001/*
002 * Copyright 2009-2021 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2009-2021 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2009-2021 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.unboundidds.examples;
037
038
039
040import java.io.File;
041import java.io.FileInputStream;
042import java.io.InputStream;
043import java.io.InputStreamReader;
044import java.io.IOException;
045import java.io.OutputStream;
046import java.io.Serializable;
047import java.text.DecimalFormat;
048import java.util.ArrayList;
049import java.util.HashMap;
050import java.util.HashSet;
051import java.util.Iterator;
052import java.util.LinkedHashMap;
053import java.util.List;
054import java.util.Map;
055import java.util.TreeMap;
056import java.util.concurrent.atomic.AtomicLong;
057import java.util.zip.GZIPInputStream;
058import javax.crypto.BadPaddingException;
059
060import com.unboundid.ldap.sdk.DN;
061import com.unboundid.ldap.sdk.Filter;
062import com.unboundid.ldap.sdk.LDAPException;
063import com.unboundid.ldap.sdk.RDN;
064import com.unboundid.ldap.sdk.ResultCode;
065import com.unboundid.ldap.sdk.SearchScope;
066import com.unboundid.ldap.sdk.Version;
067import com.unboundid.ldap.sdk.unboundidds.logs.AbandonRequestAccessLogMessage;
068import com.unboundid.ldap.sdk.unboundidds.logs.AccessLogMessage;
069import com.unboundid.ldap.sdk.unboundidds.logs.AccessLogReader;
070import com.unboundid.ldap.sdk.unboundidds.logs.AddResultAccessLogMessage;
071import com.unboundid.ldap.sdk.unboundidds.logs.BindResultAccessLogMessage;
072import com.unboundid.ldap.sdk.unboundidds.logs.CompareResultAccessLogMessage;
073import com.unboundid.ldap.sdk.unboundidds.logs.ConnectAccessLogMessage;
074import com.unboundid.ldap.sdk.unboundidds.logs.DeleteResultAccessLogMessage;
075import com.unboundid.ldap.sdk.unboundidds.logs.DisconnectAccessLogMessage;
076import com.unboundid.ldap.sdk.unboundidds.logs.ExtendedRequestAccessLogMessage;
077import com.unboundid.ldap.sdk.unboundidds.logs.ExtendedResultAccessLogMessage;
078import com.unboundid.ldap.sdk.unboundidds.logs.LogException;
079import com.unboundid.ldap.sdk.unboundidds.logs.ModifyDNResultAccessLogMessage;
080import com.unboundid.ldap.sdk.unboundidds.logs.ModifyResultAccessLogMessage;
081import com.unboundid.ldap.sdk.unboundidds.logs.OperationAccessLogMessage;
082import com.unboundid.ldap.sdk.unboundidds.logs.SearchRequestAccessLogMessage;
083import com.unboundid.ldap.sdk.unboundidds.logs.SearchResultAccessLogMessage;
084import com.unboundid.ldap.sdk.unboundidds.logs.
085            SecurityNegotiationAccessLogMessage;
086import com.unboundid.ldap.sdk.unboundidds.logs.UnbindRequestAccessLogMessage;
087import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
088import com.unboundid.util.CommandLineTool;
089import com.unboundid.util.Debug;
090import com.unboundid.util.NotMutable;
091import com.unboundid.util.NotNull;
092import com.unboundid.util.Nullable;
093import com.unboundid.util.ObjectPair;
094import com.unboundid.util.ReverseComparator;
095import com.unboundid.util.StaticUtils;
096import com.unboundid.util.ThreadSafety;
097import com.unboundid.util.ThreadSafetyLevel;
098import com.unboundid.util.args.ArgumentException;
099import com.unboundid.util.args.ArgumentParser;
100import com.unboundid.util.args.BooleanArgument;
101import com.unboundid.util.args.DurationArgument;
102import com.unboundid.util.args.FileArgument;
103import com.unboundid.util.args.IntegerArgument;
104
105
106
107/**
108 * This class provides a tool that may be used to read and summarize the
109 * contents of one or more access log files from Ping Identity, UnboundID and
110 * Nokia/Alcatel-Lucent 8661 server products.
111 * <BR>
112 * <BLOCKQUOTE>
113 *   <B>NOTE:</B>  This class, and other classes within the
114 *   {@code com.unboundid.ldap.sdk.unboundidds} package structure, are only
115 *   supported for use against Ping Identity, UnboundID, and
116 *   Nokia/Alcatel-Lucent 8661 server products.  These classes provide support
117 *   for proprietary functionality or for external specifications that are not
118 *   considered stable or mature enough to be guaranteed to work in an
119 *   interoperable way with other types of LDAP servers.
120 * </BLOCKQUOTE>
121 * Information that will be reported includes:
122 * <UL>
123 *   <LI>The total length of time covered by the log files.</LI>
124 *   <LI>The number of connections established and disconnected, the addresses
125 *       of the most commonly-connecting clients, and the average rate of
126 *       connects and disconnects per second.</LI>
127 *   <LI>The number of operations processed, overall and by operation type,
128 *       and the average rate of operations per second.</LI>
129 *   <LI>The average duration for operations processed, overall and by operation
130 *       type.</LI>
131 *   <LI>A breakdown of operation processing times into a number of predefined
132 *       categories, ranging from less than one millisecond to over one
133 *       minute.</LI>
134 *   <LI>A breakdown of the most common result codes for each type of operation
135 *       and their relative frequencies.</LI>
136 *   <LI>The most common types of extended requests processed and their
137 *       relative frequencies.</LI>
138 *   <LI>The number of unindexed search operations processed and the most common
139 *       types of filters used in unindexed searches.</LI>
140 *   <LI>A breakdown of the relative frequencies for each type of search
141 *       scope.</LI>
142 *   <LI>The most common types of search filters used for search
143 *       operations and their relative frequencies.</LI>
144 * </UL>
145 * It is designed to work with access log files using either the default log
146 * format with separate request and response messages, as well as log files
147 * in which the request and response details have been combined on the same
148 * line.  The log files to be processed should be provided as command-line
149 * arguments.
150 * <BR><BR>
151 * The APIs demonstrated by this example include:
152 * <UL>
153 *   <LI>Access log parsing (from the
154 *       {@code com.unboundid.ldap.sdk.unboundidds.logs} package)</LI>
155 *   <LI>Argument parsing (from the {@code com.unboundid.util.args}
156 *       package)</LI>
157 * </UL>
158 */
159@NotMutable()
160@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
161public final class SummarizeAccessLog
162       extends CommandLineTool
163       implements Serializable
164{
165  /**
166   * The serial version UID for this serializable class.
167   */
168  private static final long serialVersionUID = 7189168366509887130L;
169
170
171
172  // Variables used for accessing argument information.
173  @Nullable private ArgumentParser argumentParser;
174
175  // An argument that may be used to indicate that the summarized output should
176  // not be anonymized, and should include attribute values.
177  @Nullable private BooleanArgument doNotAnonymize;
178
179  // An argument that may be used to indicate that the log files are compressed.
180  @Nullable private BooleanArgument isCompressed;
181
182  // An argument used to specify the encryption passphrase.
183  @Nullable private FileArgument encryptionPassphraseFile;
184
185  // An argument used to specify the maximum number of values to report for each
186  // item.
187  @Nullable private IntegerArgument reportCount;
188
189  // The decimal format that will be used for this class.
190  @NotNull private final DecimalFormat decimalFormat;
191
192  // The total duration for log content, in milliseconds.
193  private long logDurationMillis;
194
195  // The total processing time for each type of operation.
196  private double addProcessingDuration;
197  private double bindProcessingDuration;
198  private double compareProcessingDuration;
199  private double deleteProcessingDuration;
200  private double extendedProcessingDuration;
201  private double modifyProcessingDuration;
202  private double modifyDNProcessingDuration;
203  private double searchProcessingDuration;
204
205  // A variable used for counting the number of messages of each type.
206  private long numAbandons;
207  private long numAdds;
208  private long numBinds;
209  private long numCompares;
210  private long numConnects;
211  private long numDeletes;
212  private long numDisconnects;
213  private long numExtended;
214  private long numModifies;
215  private long numModifyDNs;
216  private long numSearches;
217  private long numUnbinds;
218
219  // The number of operations of each type that accessed uncached data.
220  private long numUncachedAdds;
221  private long numUncachedBinds;
222  private long numUncachedCompares;
223  private long numUncachedDeletes;
224  private long numUncachedExtended;
225  private long numUncachedModifies;
226  private long numUncachedModifyDNs;
227  private long numUncachedSearches;
228
229  // The number of unindexed searches processed within the server.
230  private long numUnindexedAttempts;
231  private long numUnindexedFailed;
232  private long numUnindexedSuccessful;
233
234  // Variables used for maintaining counts for common types of information.
235  @NotNull private final HashMap<Long,AtomicLong> searchEntryCounts;
236  @NotNull private final HashMap<ResultCode,AtomicLong> addResultCodes;
237  @NotNull private final HashMap<ResultCode,AtomicLong> bindResultCodes;
238  @NotNull private final HashMap<ResultCode,AtomicLong> compareResultCodes;
239  @NotNull private final HashMap<ResultCode,AtomicLong> deleteResultCodes;
240  @NotNull private final HashMap<ResultCode,AtomicLong> extendedResultCodes;
241  @NotNull private final HashMap<ResultCode,AtomicLong> modifyResultCodes;
242  @NotNull private final HashMap<ResultCode,AtomicLong> modifyDNResultCodes;
243  @NotNull private final HashMap<ResultCode,AtomicLong> searchResultCodes;
244  @NotNull private final HashMap<SearchScope,AtomicLong> searchScopes;
245  @NotNull private final HashMap<String,AtomicLong> authenticationTypes;
246  @NotNull private final HashMap<String,AtomicLong> authzDNs;
247  @NotNull private final HashMap<String,AtomicLong> failedBindDNs;
248  @NotNull private final HashMap<String,AtomicLong> successfulBindDNs;
249  @NotNull private final HashMap<String,AtomicLong> clientAddresses;
250  @NotNull private final HashMap<String,AtomicLong> clientConnectionPolicies;
251  @NotNull private final HashMap<String,AtomicLong> disconnectReasons;
252  @NotNull private final HashMap<String,AtomicLong> extendedOperations;
253  @NotNull private final HashMap<String,AtomicLong> filterTypes;
254  @NotNull private final HashMap<String,AtomicLong> mostExpensiveFilters;
255  @NotNull private final HashMap<String,AtomicLong> multiEntryFilters;
256  @NotNull private final HashMap<String,AtomicLong> noEntryFilters;
257  @NotNull private final HashMap<String,AtomicLong> oneEntryFilters;
258  @NotNull private final HashMap<String,AtomicLong> searchBaseDNs;
259  @NotNull private final HashMap<String,AtomicLong> tlsCipherSuites;
260  @NotNull private final HashMap<String,AtomicLong> tlsProtocols;
261  @NotNull private final HashMap<String,AtomicLong> unindexedFilters;
262  @NotNull private final HashMap<String,String> extendedOperationOIDsToNames;
263  @NotNull private final HashSet<String> processedRequests;
264  @NotNull private final LinkedHashMap<Long,AtomicLong> addProcessingTimes;
265  @NotNull private final LinkedHashMap<Long,AtomicLong> bindProcessingTimes;
266  @NotNull private final LinkedHashMap<Long,AtomicLong> compareProcessingTimes;
267  @NotNull private final LinkedHashMap<Long,AtomicLong> deleteProcessingTimes;
268  @NotNull private final LinkedHashMap<Long,AtomicLong> extendedProcessingTimes;
269  @NotNull private final LinkedHashMap<Long,AtomicLong> modifyProcessingTimes;
270  @NotNull private final LinkedHashMap<Long,AtomicLong> modifyDNProcessingTimes;
271  @NotNull private final LinkedHashMap<Long,AtomicLong> searchProcessingTimes;
272
273
274
275  /**
276   * Parse the provided command line arguments and perform the appropriate
277   * processing.
278   *
279   * @param  args  The command line arguments provided to this program.
280   */
281  public static void main(@NotNull final String[] args)
282  {
283    final ResultCode resultCode = main(args, System.out, System.err);
284    if (resultCode != ResultCode.SUCCESS)
285    {
286      System.exit(resultCode.intValue());
287    }
288  }
289
290
291
292  /**
293   * Parse the provided command line arguments and perform the appropriate
294   * processing.
295   *
296   * @param  args       The command line arguments provided to this program.
297   * @param  outStream  The output stream to which standard out should be
298   *                    written.  It may be {@code null} if output should be
299   *                    suppressed.
300   * @param  errStream  The output stream to which standard error should be
301   *                    written.  It may be {@code null} if error messages
302   *                    should be suppressed.
303   *
304   * @return  A result code indicating whether the processing was successful.
305   */
306  @NotNull()
307  public static ResultCode main(@NotNull final String[] args,
308                                @Nullable final OutputStream outStream,
309                                @Nullable final OutputStream errStream)
310  {
311    final SummarizeAccessLog summarizer =
312         new SummarizeAccessLog(outStream, errStream);
313    return summarizer.runTool(args);
314  }
315
316
317
318  /**
319   * Creates a new instance of this tool.
320   *
321   * @param  outStream  The output stream to which standard out should be
322   *                    written.  It may be {@code null} if output should be
323   *                    suppressed.
324   * @param  errStream  The output stream to which standard error should be
325   *                    written.  It may be {@code null} if error messages
326   *                    should be suppressed.
327   */
328  public SummarizeAccessLog(@Nullable final OutputStream outStream,
329                            @Nullable final OutputStream errStream)
330  {
331    super(outStream, errStream);
332
333    argumentParser = null;
334    doNotAnonymize = null;
335    isCompressed = null;
336    encryptionPassphraseFile = null;
337    reportCount = null;
338
339    decimalFormat = new DecimalFormat("0.000");
340
341    logDurationMillis = 0L;
342
343    addProcessingDuration = 0.0;
344    bindProcessingDuration = 0.0;
345    compareProcessingDuration = 0.0;
346    deleteProcessingDuration = 0.0;
347    extendedProcessingDuration = 0.0;
348    modifyProcessingDuration = 0.0;
349    modifyDNProcessingDuration = 0.0;
350    searchProcessingDuration = 0.0;
351
352    numAbandons = 0L;
353    numAdds = 0L;
354    numBinds = 0L;
355    numCompares = 0L;
356    numConnects = 0L;
357    numDeletes = 0L;
358    numDisconnects = 0L;
359    numExtended = 0L;
360    numModifies = 0L;
361    numModifyDNs = 0L;
362    numSearches = 0L;
363    numUnbinds = 0L;
364
365    numUncachedAdds = 0L;
366    numUncachedBinds = 0L;
367    numUncachedCompares = 0L;
368    numUncachedDeletes = 0L;
369    numUncachedExtended = 0L;
370    numUncachedModifies = 0L;
371    numUncachedModifyDNs = 0L;
372    numUncachedSearches = 0L;
373
374    numUnindexedAttempts = 0L;
375    numUnindexedFailed = 0L;
376    numUnindexedSuccessful = 0L;
377
378    searchEntryCounts = new HashMap<>(StaticUtils.computeMapCapacity(10));
379    addResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
380    bindResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
381    compareResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
382    deleteResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
383    extendedResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
384    modifyResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
385    modifyDNResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
386    searchResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
387    searchScopes = new HashMap<>(StaticUtils.computeMapCapacity(4));
388    authenticationTypes = new HashMap<>(StaticUtils.computeMapCapacity(100));
389    authzDNs = new HashMap<>(StaticUtils.computeMapCapacity(100));
390    failedBindDNs = new HashMap<>(StaticUtils.computeMapCapacity(100));
391    successfulBindDNs = new HashMap<>(StaticUtils.computeMapCapacity(100));
392    clientAddresses = new HashMap<>(StaticUtils.computeMapCapacity(100));
393    clientConnectionPolicies =
394         new HashMap<>(StaticUtils.computeMapCapacity(100));
395    disconnectReasons = new HashMap<>(StaticUtils.computeMapCapacity(100));
396    extendedOperations = new HashMap<>(StaticUtils.computeMapCapacity(10));
397    filterTypes = new HashMap<>(StaticUtils.computeMapCapacity(100));
398    mostExpensiveFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
399    multiEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
400    noEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
401    oneEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
402    searchBaseDNs = new HashMap<>(StaticUtils.computeMapCapacity(100));
403    tlsCipherSuites = new HashMap<>(StaticUtils.computeMapCapacity(100));
404    tlsProtocols = new HashMap<>(StaticUtils.computeMapCapacity(100));
405    unindexedFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
406    extendedOperationOIDsToNames =
407         new HashMap<>(StaticUtils.computeMapCapacity(100));
408    processedRequests = new HashSet<>(StaticUtils.computeMapCapacity(100));
409    addProcessingTimes =
410         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
411    bindProcessingTimes =
412         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
413    compareProcessingTimes =
414         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
415    deleteProcessingTimes =
416         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
417    extendedProcessingTimes =
418         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
419    modifyProcessingTimes =
420         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
421    modifyDNProcessingTimes =
422         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
423    searchProcessingTimes =
424         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
425
426    populateProcessingTimeMap(addProcessingTimes);
427    populateProcessingTimeMap(bindProcessingTimes);
428    populateProcessingTimeMap(compareProcessingTimes);
429    populateProcessingTimeMap(deleteProcessingTimes);
430    populateProcessingTimeMap(extendedProcessingTimes);
431    populateProcessingTimeMap(modifyProcessingTimes);
432    populateProcessingTimeMap(modifyDNProcessingTimes);
433    populateProcessingTimeMap(searchProcessingTimes);
434  }
435
436
437
438  /**
439   * Retrieves the name for this tool.
440   *
441   * @return  The name for this tool.
442   */
443  @Override()
444  @NotNull()
445  public String getToolName()
446  {
447    return "summarize-access-log";
448  }
449
450
451
452  /**
453   * Retrieves the description for this tool.
454   *
455   * @return  The description for this tool.
456   */
457  @Override()
458  @NotNull()
459  public String getToolDescription()
460  {
461    return "Examine one or more access log files from Ping Identity, " +
462         "UnboundID, or Nokia/Alcatel-Lucent 8661 server products to display " +
463         "a number of metrics about operations processed within the server.";
464  }
465
466
467
468  /**
469   * Retrieves the version string for this tool.
470   *
471   * @return  The version string for this tool.
472   */
473  @Override()
474  @NotNull()
475  public String getToolVersion()
476  {
477    return Version.NUMERIC_VERSION_STRING;
478  }
479
480
481
482  /**
483   * Retrieves the minimum number of unnamed trailing arguments that are
484   * required.
485   *
486   * @return  One, to indicate that at least one trailing argument (representing
487   *          the path to an access log file) must be provided.
488   */
489  @Override()
490  public int getMinTrailingArguments()
491  {
492    return 1;
493  }
494
495
496
497  /**
498   * Retrieves the maximum number of unnamed trailing arguments that may be
499   * provided for this tool.
500   *
501   * @return  The maximum number of unnamed trailing arguments that may be
502   *          provided for this tool.
503   */
504  @Override()
505  public int getMaxTrailingArguments()
506  {
507    return -1;
508  }
509
510
511
512  /**
513   * Retrieves a placeholder string that should be used for trailing arguments
514   * in the usage information for this tool.
515   *
516   * @return  A placeholder string that should be used for trailing arguments in
517   *          the usage information for this tool.
518   */
519  @Override()
520  @NotNull()
521  public String getTrailingArgumentsPlaceholder()
522  {
523    return "{path}";
524  }
525
526
527
528  /**
529   * Indicates whether this tool should provide support for an interactive mode,
530   * in which the tool offers a mode in which the arguments can be provided in
531   * a text-driven menu rather than requiring them to be given on the command
532   * line.  If interactive mode is supported, it may be invoked using the
533   * "--interactive" argument.  Alternately, if interactive mode is supported
534   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
535   * interactive mode may be invoked by simply launching the tool without any
536   * arguments.
537   *
538   * @return  {@code true} if this tool supports interactive mode, or
539   *          {@code false} if not.
540   */
541  @Override()
542  public boolean supportsInteractiveMode()
543  {
544    return true;
545  }
546
547
548
549  /**
550   * Indicates whether this tool defaults to launching in interactive mode if
551   * the tool is invoked without any command-line arguments.  This will only be
552   * used if {@link #supportsInteractiveMode()} returns {@code true}.
553   *
554   * @return  {@code true} if this tool defaults to using interactive mode if
555   *          launched without any command-line arguments, or {@code false} if
556   *          not.
557   */
558  @Override()
559  public boolean defaultsToInteractiveMode()
560  {
561    return true;
562  }
563
564
565
566  /**
567   * Indicates whether this tool should provide arguments for redirecting output
568   * to a file.  If this method returns {@code true}, then the tool will offer
569   * an "--outputFile" argument that will specify the path to a file to which
570   * all standard output and standard error content will be written, and it will
571   * also offer a "--teeToStandardOut" argument that can only be used if the
572   * "--outputFile" argument is present and will cause all output to be written
573   * to both the specified output file and to standard output.
574   *
575   * @return  {@code true} if this tool should provide arguments for redirecting
576   *          output to a file, or {@code false} if not.
577   */
578  @Override()
579  protected boolean supportsOutputFile()
580  {
581    return true;
582  }
583
584
585
586  /**
587   * Indicates whether this tool supports the use of a properties file for
588   * specifying default values for arguments that aren't specified on the
589   * command line.
590   *
591   * @return  {@code true} if this tool supports the use of a properties file
592   *          for specifying default values for arguments that aren't specified
593   *          on the command line, or {@code false} if not.
594   */
595  @Override()
596  public boolean supportsPropertiesFile()
597  {
598    return true;
599  }
600
601
602
603  /**
604   * Adds the command-line arguments supported for use with this tool to the
605   * provided argument parser.  The tool may need to retain references to the
606   * arguments (and/or the argument parser, if trailing arguments are allowed)
607   * to it in order to obtain their values for use in later processing.
608   *
609   * @param  parser  The argument parser to which the arguments are to be added.
610   *
611   * @throws  ArgumentException  If a problem occurs while adding any of the
612   *                             tool-specific arguments to the provided
613   *                             argument parser.
614   */
615  @Override()
616  public void addToolArguments(@NotNull final ArgumentParser parser)
617         throws ArgumentException
618  {
619    // We need to save a reference to the argument parser so that we can get
620    // the trailing arguments later.
621    argumentParser = parser;
622
623    // Add an argument that makes it possible to read a compressed log file.
624    // Note that this argument is no longer needed for dealing with compressed
625    // files, since the tool will automatically detect whether a file is
626    // compressed.  However, the argument is still provided for the purpose of
627    // backward compatibility.
628    String description = "Indicates that the log file is compressed.";
629    isCompressed = new BooleanArgument('c', "isCompressed", description);
630    isCompressed.addLongIdentifier("is-compressed", true);
631    isCompressed.addLongIdentifier("compressed", true);
632    isCompressed.setHidden(true);
633    parser.addArgument(isCompressed);
634
635
636    // Add an argument that indicates that the tool should read the encryption
637    // passphrase from a file.
638    description = "Indicates that the log file is encrypted and that the " +
639         "encryption passphrase is contained in the specified file.  If " +
640         "the log data is encrypted and this argument is not provided, then " +
641         "the tool will interactively prompt for the encryption passphrase.";
642    encryptionPassphraseFile = new FileArgument(null,
643         "encryptionPassphraseFile", false, 1, null, description, true, true,
644         true, false);
645    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
646         true);
647    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
648    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
649         true);
650    parser.addArgument(encryptionPassphraseFile);
651
652
653    // Add an argument that indicates the number of values to display for each
654    // item being summarized.
655    description = "The number of values to display for each item being " +
656         "summarized.  A value of zero indicates that all items should be " +
657         "displayed.  If this is not provided, a default value of 20 will " +
658         "be used.";
659    reportCount = new IntegerArgument(null, "reportCount", false, 0, null,
660         description, 0, Integer.MAX_VALUE, 20);
661    reportCount.addLongIdentifier("report-count", true);
662    reportCount.addLongIdentifier("maximumCount", true);
663    reportCount.addLongIdentifier("maximum-count", true);
664    reportCount.addLongIdentifier("maxCount", true);
665    reportCount.addLongIdentifier("max-count", true);
666    reportCount.addLongIdentifier("count", true);
667    parser.addArgument(reportCount);
668
669
670    // Add an argument that indicates that the output should not be anonymized.
671    description = "Do not anonymize the output, but include actual attribute " +
672         "values in filters and DNs.  This will also have the effect of " +
673         "de-generifying those values, so output including the most common " +
674         "filters and DNs in some category will be specific instances of " +
675         "those filters and DNs instead of generic patterns.";
676    doNotAnonymize = new BooleanArgument(null, "doNotAnonymize", 1,
677         description);
678    doNotAnonymize.addLongIdentifier("do-not-anonymize", true);
679    doNotAnonymize.addLongIdentifier("deAnonymize", true);
680    doNotAnonymize.addLongIdentifier("de-anonymize", true);
681    parser.addArgument(doNotAnonymize);
682  }
683
684
685
686  /**
687   * Performs any necessary processing that should be done to ensure that the
688   * provided set of command-line arguments were valid.  This method will be
689   * called after the basic argument parsing has been performed and immediately
690   * before the {@link #doToolProcessing} method is invoked.
691   *
692   * @throws  ArgumentException  If there was a problem with the command-line
693   *                             arguments provided to this program.
694   */
695  @Override()
696  public void doExtendedArgumentValidation()
697         throws ArgumentException
698  {
699    // Make sure that at least one access log file path was provided.
700    final List<String> trailingArguments =
701         argumentParser.getTrailingArguments();
702    if ((trailingArguments == null) || trailingArguments.isEmpty())
703    {
704      throw new ArgumentException("No access log file paths were provided.");
705    }
706  }
707
708
709
710  /**
711   * Performs the core set of processing for this tool.
712   *
713   * @return  A result code that indicates whether the processing completed
714   *          successfully.
715   */
716  @Override()
717  @NotNull()
718  public ResultCode doToolProcessing()
719  {
720    int displayCount = reportCount.getValue();
721    if (displayCount <= 0)
722    {
723      displayCount = Integer.MAX_VALUE;
724    }
725
726    String encryptionPassphrase = null;
727    if (encryptionPassphraseFile.isPresent())
728    {
729      try
730      {
731        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
732             encryptionPassphraseFile.getValue());
733      }
734      catch (final LDAPException e)
735      {
736        Debug.debugException(e);
737        err(e.getMessage());
738        return e.getResultCode();
739      }
740    }
741
742
743    long logLines = 0L;
744    for (final String path : argumentParser.getTrailingArguments())
745    {
746      final File f = new File(path);
747      out("Examining access log ", f.getAbsolutePath());
748      AccessLogReader reader = null;
749      InputStream inputStream = null;
750      try
751      {
752        inputStream = new FileInputStream(f);
753
754        final ObjectPair<InputStream,String> p =
755             ToolUtils.getPossiblyPassphraseEncryptedInputStream(inputStream,
756                  encryptionPassphrase,
757                  (! encryptionPassphraseFile.isPresent()),
758                  "Log file '" + path + "' is encrypted.  Please enter the " +
759                       "encryption passphrase:",
760                  "ERROR:  The provided passphrase was incorrect.",
761                  getOut(), getErr());
762        inputStream = p.getFirst();
763        if ((p.getSecond() != null) && (encryptionPassphrase == null))
764        {
765          encryptionPassphrase = p.getSecond();
766        }
767
768        if (isCompressed.isPresent())
769        {
770          inputStream = new GZIPInputStream(inputStream);
771        }
772        else
773        {
774          inputStream =
775               ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream);
776        }
777
778        reader = new AccessLogReader(new InputStreamReader(inputStream));
779      }
780      catch (final Exception e)
781      {
782        Debug.debugException(e);
783        err("Unable to open access log file ", f.getAbsolutePath(), ":  ",
784            StaticUtils.getExceptionMessage(e));
785        return ResultCode.LOCAL_ERROR;
786      }
787      finally
788      {
789        if ((reader == null) && (inputStream != null))
790        {
791          try
792          {
793            inputStream.close();
794          }
795          catch (final Exception e)
796          {
797            Debug.debugException(e);
798          }
799        }
800      }
801
802      long startTime = 0L;
803      long stopTime  = 0L;
804
805      while (true)
806      {
807        final AccessLogMessage msg;
808        try
809        {
810          msg = reader.read();
811        }
812        catch (final IOException ioe)
813        {
814          Debug.debugException(ioe);
815          err("Error reading from access log file ", f.getAbsolutePath(),
816              ":  ", StaticUtils.getExceptionMessage(ioe));
817
818          if ((ioe.getCause() != null) &&
819               (ioe.getCause() instanceof BadPaddingException))
820          {
821            err("This error is likely because the log is encrypted and the " +
822                 "server still has the log file open.  It is recommended " +
823                 "that you only try to examine encrypted logs after they " +
824                 "have been rotated.  You can use the rotate-log tool to " +
825                 "force a rotation at any time.  Attempting to proceed with " +
826                 "just the data that was successfully read.");
827            break;
828          }
829          else
830          {
831            return ResultCode.LOCAL_ERROR;
832          }
833        }
834        catch (final LogException le)
835        {
836          Debug.debugException(le);
837          err("Encountered an error while attempting to parse a line in" +
838              "access log file ", f.getAbsolutePath(), ":  ",
839              StaticUtils.getExceptionMessage(le));
840          continue;
841        }
842
843        if (msg == null)
844        {
845          break;
846        }
847
848        logLines++;
849        stopTime = msg.getTimestamp().getTime();
850        if (startTime == 0L)
851        {
852          startTime = stopTime;
853        }
854
855        switch (msg.getMessageType())
856        {
857          case CONNECT:
858            processConnect((ConnectAccessLogMessage) msg);
859            break;
860          case SECURITY_NEGOTIATION:
861            processSecurityNegotiation(
862                 (SecurityNegotiationAccessLogMessage) msg);
863            break;
864          case DISCONNECT:
865            processDisconnect((DisconnectAccessLogMessage) msg);
866            break;
867          case REQUEST:
868            switch (((OperationAccessLogMessage) msg).getOperationType())
869            {
870              case ABANDON:
871                processAbandonRequest((AbandonRequestAccessLogMessage) msg);
872                break;
873              case EXTENDED:
874                processExtendedRequest((ExtendedRequestAccessLogMessage) msg);
875                break;
876              case SEARCH:
877                processSearchRequest((SearchRequestAccessLogMessage) msg);
878                break;
879              case UNBIND:
880                processUnbindRequest((UnbindRequestAccessLogMessage) msg);
881                break;
882            }
883            break;
884          case RESULT:
885            switch (((OperationAccessLogMessage) msg).getOperationType())
886            {
887              case ADD:
888                processAddResult((AddResultAccessLogMessage) msg);
889                break;
890              case BIND:
891                processBindResult((BindResultAccessLogMessage) msg);
892                break;
893              case COMPARE:
894                processCompareResult((CompareResultAccessLogMessage) msg);
895                break;
896              case DELETE:
897                processDeleteResult((DeleteResultAccessLogMessage) msg);
898                break;
899              case EXTENDED:
900                processExtendedResult((ExtendedResultAccessLogMessage) msg);
901                break;
902              case MODIFY:
903                processModifyResult((ModifyResultAccessLogMessage) msg);
904                break;
905              case MODDN:
906                processModifyDNResult((ModifyDNResultAccessLogMessage) msg);
907                break;
908              case SEARCH:
909                processSearchResult((SearchResultAccessLogMessage) msg);
910                break;
911            }
912            break;
913
914          case ASSURANCE_COMPLETE:
915          case CLIENT_CERTIFICATE:
916          case ENTRY_REBALANCING_REQUEST:
917          case ENTRY_REBALANCING_RESULT:
918          case FORWARD:
919          case FORWARD_FAILED:
920          case ENTRY:
921          case REFERENCE:
922          default:
923            // Nothing needs to be done for these message types.
924        }
925      }
926
927      try
928      {
929        reader.close();
930      }
931      catch (final Exception e)
932      {
933        Debug.debugException(e);
934      }
935      logDurationMillis += (stopTime - startTime);
936    }
937
938
939    final int numFiles = argumentParser.getTrailingArguments().size();
940    out();
941    out("Examined ", logLines, " lines in ", numFiles,
942        ((numFiles == 1) ? " file" : " files"),
943        " covering a total duration of ",
944        StaticUtils.millisToHumanReadableDuration(logDurationMillis));
945    if (logLines == 0)
946    {
947      return ResultCode.SUCCESS;
948    }
949
950    out();
951
952    final double logDurationSeconds   = logDurationMillis / 1_000.0;
953    final double connectsPerSecond    = numConnects / logDurationSeconds;
954    final double disconnectsPerSecond = numDisconnects / logDurationSeconds;
955
956    out("Total connections established:  ", numConnects, " (",
957        decimalFormat.format(connectsPerSecond), "/second)");
958    out("Total disconnects:  ", numDisconnects, " (",
959        decimalFormat.format(disconnectsPerSecond), "/second)");
960
961    printCounts(clientAddresses, "Most common client addresses:", "address",
962         "addresses");
963
964    printCounts(clientConnectionPolicies,
965         "Most common client connection policies:", "policy", "policies");
966
967    printCounts(tlsProtocols, "Most common TLS protocol versions:", "version",
968         "versions");
969
970    printCounts(tlsCipherSuites, "Most common TLS cipher suites:",
971         "cipher suite", "cipher suites");
972
973    printCounts(disconnectReasons, "Most common disconnect reasons:", "reason",
974         "reasons");
975
976    final long totalOps = numAbandons + numAdds + numBinds + numCompares +
977         numDeletes + numExtended + numModifies + numModifyDNs + numSearches +
978         numUnbinds;
979    if (totalOps > 0)
980    {
981      final double percentAbandon  = 100.0 * numAbandons / totalOps;
982      final double percentAdd      = 100.0 * numAdds / totalOps;
983      final double percentBind     = 100.0 * numBinds / totalOps;
984      final double percentCompare  = 100.0 * numCompares / totalOps;
985      final double percentDelete   = 100.0 * numDeletes / totalOps;
986      final double percentExtended = 100.0 * numExtended / totalOps;
987      final double percentModify   = 100.0 * numModifies / totalOps;
988      final double percentModifyDN = 100.0 * numModifyDNs / totalOps;
989      final double percentSearch   = 100.0 * numSearches / totalOps;
990      final double percentUnbind   = 100.0 * numUnbinds / totalOps;
991
992      final double abandonsPerSecond  = numAbandons / logDurationSeconds;
993      final double addsPerSecond      = numAdds / logDurationSeconds;
994      final double bindsPerSecond     = numBinds / logDurationSeconds;
995      final double comparesPerSecond  = numCompares / logDurationSeconds;
996      final double deletesPerSecond   = numDeletes / logDurationSeconds;
997      final double extendedPerSecond  = numExtended / logDurationSeconds;
998      final double modifiesPerSecond  = numModifies / logDurationSeconds;
999      final double modifyDNsPerSecond = numModifyDNs / logDurationSeconds;
1000      final double searchesPerSecond  = numSearches / logDurationSeconds;
1001      final double unbindsPerSecond   = numUnbinds / logDurationSeconds;
1002
1003      out();
1004      out("Total operations examined:  ", totalOps);
1005      out("Abandon operations examined:  ", numAbandons, " (",
1006          decimalFormat.format(percentAbandon), "%, ",
1007          decimalFormat.format(abandonsPerSecond), "/second)");
1008      out("Add operations examined:  ", numAdds, " (",
1009          decimalFormat.format(percentAdd), "%, ",
1010          decimalFormat.format(addsPerSecond), "/second)");
1011      out("Bind operations examined:  ", numBinds, " (",
1012          decimalFormat.format(percentBind), "%, ",
1013          decimalFormat.format(bindsPerSecond), "/second)");
1014      out("Compare operations examined:  ", numCompares, " (",
1015          decimalFormat.format(percentCompare), "%, ",
1016          decimalFormat.format(comparesPerSecond), "/second)");
1017      out("Delete operations examined:  ", numDeletes, " (",
1018          decimalFormat.format(percentDelete), "%, ",
1019          decimalFormat.format(deletesPerSecond), "/second)");
1020      out("Extended operations examined:  ", numExtended, " (",
1021          decimalFormat.format(percentExtended), "%, ",
1022          decimalFormat.format(extendedPerSecond), "/second)");
1023      out("Modify operations examined:  ", numModifies, " (",
1024          decimalFormat.format(percentModify), "%, ",
1025          decimalFormat.format(modifiesPerSecond), "/second)");
1026      out("Modify DN operations examined:  ", numModifyDNs, " (",
1027          decimalFormat.format(percentModifyDN), "%, ",
1028          decimalFormat.format(modifyDNsPerSecond), "/second)");
1029      out("Search operations examined:  ", numSearches, " (",
1030          decimalFormat.format(percentSearch), "%, ",
1031          decimalFormat.format(searchesPerSecond), "/second)");
1032      out("Unbind operations examined:  ", numUnbinds, " (",
1033          decimalFormat.format(percentUnbind), "%, ",
1034          decimalFormat.format(unbindsPerSecond), "/second)");
1035
1036      final double totalProcessingDuration = addProcessingDuration +
1037           bindProcessingDuration + compareProcessingDuration +
1038           deleteProcessingDuration + extendedProcessingDuration +
1039           modifyProcessingDuration + modifyDNProcessingDuration +
1040           searchProcessingDuration;
1041
1042      out();
1043      out("Average operation processing duration:  ",
1044          decimalFormat.format(totalProcessingDuration / totalOps), "ms");
1045
1046      if (numAdds > 0)
1047      {
1048        out("Average add operation processing duration:  ",
1049            decimalFormat.format(addProcessingDuration / numAdds), "ms");
1050      }
1051
1052      if (numBinds > 0)
1053      {
1054        out("Average bind operation processing duration:  ",
1055            decimalFormat.format(bindProcessingDuration / numBinds), "ms");
1056      }
1057
1058      if (numCompares > 0)
1059      {
1060        out("Average compare operation processing duration:  ",
1061            decimalFormat.format(compareProcessingDuration / numCompares),
1062            "ms");
1063      }
1064
1065      if (numDeletes > 0)
1066      {
1067        out("Average delete operation processing duration:  ",
1068            decimalFormat.format(deleteProcessingDuration / numDeletes), "ms");
1069      }
1070
1071      if (numExtended > 0)
1072      {
1073        out("Average extended operation processing duration:  ",
1074            decimalFormat.format(extendedProcessingDuration / numExtended),
1075            "ms");
1076      }
1077
1078      if (numModifies > 0)
1079      {
1080        out("Average modify operation processing duration:  ",
1081            decimalFormat.format(modifyProcessingDuration / numModifies), "ms");
1082      }
1083
1084      if (numModifyDNs > 0)
1085      {
1086        out("Average modify DN operation processing duration:  ",
1087            decimalFormat.format(modifyDNProcessingDuration / numModifyDNs),
1088            "ms");
1089      }
1090
1091      if (numSearches > 0)
1092      {
1093        out("Average search operation processing duration:  ",
1094            decimalFormat.format(searchProcessingDuration / numSearches), "ms");
1095      }
1096
1097      printProcessingTimeHistogram("add", numAdds, addProcessingTimes);
1098      printProcessingTimeHistogram("bind", numBinds, bindProcessingTimes);
1099      printProcessingTimeHistogram("compare", numCompares,
1100                                   compareProcessingTimes);
1101      printProcessingTimeHistogram("delete", numDeletes, deleteProcessingTimes);
1102      printProcessingTimeHistogram("extended", numExtended,
1103                                   extendedProcessingTimes);
1104      printProcessingTimeHistogram("modify", numModifies,
1105                                   modifyProcessingTimes);
1106      printProcessingTimeHistogram("modify DN", numModifyDNs,
1107                                 modifyDNProcessingTimes);
1108      printProcessingTimeHistogram("search", numSearches,
1109                                   searchProcessingTimes);
1110
1111      printResultCodeCounts(addResultCodes, "add");
1112      printResultCodeCounts(bindResultCodes, "bind");
1113      printResultCodeCounts(compareResultCodes, "compare");
1114      printResultCodeCounts(deleteResultCodes, "delete");
1115      printResultCodeCounts(extendedResultCodes, "extended");
1116      printResultCodeCounts(modifyResultCodes, "modify");
1117      printResultCodeCounts(modifyDNResultCodes, "modify DN");
1118      printResultCodeCounts(searchResultCodes, "search");
1119
1120      printCounts(successfulBindDNs,
1121           "Most common bind DNs used in successful authentication attempts:",
1122           "DN", "DNs");
1123      printCounts(failedBindDNs,
1124           "Most common bind DNs used in failed authentication attempts:",
1125           "DN", "DNs");
1126      printCounts(authenticationTypes, "Most common authentication types:",
1127           "authentication type", "authentication types");
1128
1129      long numResultsWithAuthzID = 0L;
1130      for (final AtomicLong l : authzDNs.values())
1131      {
1132        numResultsWithAuthzID += l.get();
1133      }
1134
1135      out();
1136      final double percentWithAuthzID =
1137           100.0 * numResultsWithAuthzID / totalOps;
1138      out("Number of operations with an alternate authorization identity:  ",
1139           numResultsWithAuthzID, " (",
1140           decimalFormat.format(percentWithAuthzID), "%)");
1141
1142      printCounts(authzDNs, "Most common alternate authorization identity DNs:",
1143           "DN", "DNs");
1144
1145      if (! extendedOperations.isEmpty())
1146      {
1147        final List<ObjectPair<String,Long>> extOpCounts = new ArrayList<>();
1148        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1149        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1150        getMostCommonElements(extendedOperations, extOpCounts, displayCount,
1151             skippedWithSameCount, skippedWithLowerCount);
1152
1153        out();
1154        out("Most common extended operation types:");
1155
1156        long count = -1L;
1157        for (final ObjectPair<String,Long> p : extOpCounts)
1158        {
1159          count = p.getSecond();
1160          final double percent = 100.0 * count / numExtended;
1161
1162          final String oid = p.getFirst();
1163          final String name = extendedOperationOIDsToNames.get(oid);
1164          if (name == null)
1165          {
1166            out(p.getFirst(), ":  ", p.getSecond(), " (",
1167                 decimalFormat.format(percent), "%)");
1168          }
1169          else
1170          {
1171            out(p.getFirst(), " (", name, "):  ", p.getSecond(), " (",
1172                 decimalFormat.format(percent), "%)");
1173          }
1174        }
1175
1176        if (skippedWithSameCount.get() > 0L)
1177        {
1178          out("{ Skipped " + skippedWithSameCount.get() +
1179               " additional extended " +
1180               getSingularOrPlural(skippedWithSameCount.get(), "operation",
1181                    "operations") +
1182               " with a count of " + count + " }");
1183        }
1184
1185        if (skippedWithLowerCount.get() > 0L)
1186        {
1187          out("{ Skipped " + skippedWithLowerCount.get() +
1188               " additional extended " +
1189               getSingularOrPlural(skippedWithLowerCount.get(), "operation",
1190                    "operations") +
1191               " with a count that is less than " + count + " }");
1192        }
1193      }
1194
1195      out();
1196      out("Number of unindexed search attempts:  ", numUnindexedAttempts);
1197      out("Number of successfully-completed unindexed searches:  ",
1198           numUnindexedSuccessful);
1199      out("Number of failed unindexed searches:  ", numUnindexedFailed);
1200
1201      printCounts(unindexedFilters, "Most common unindexed search filters:",
1202           "filter", "filters");
1203
1204      if (! searchScopes.isEmpty())
1205      {
1206        final List<ObjectPair<SearchScope,Long>> scopeCounts =
1207             new ArrayList<>();
1208        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1209        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1210        getMostCommonElements(searchScopes, scopeCounts, displayCount,
1211             skippedWithSameCount, skippedWithLowerCount);
1212
1213        out();
1214        out("Most common search scopes:");
1215
1216        long count = -1L;
1217        for (final ObjectPair<SearchScope,Long> p : scopeCounts)
1218        {
1219          count = p.getSecond();
1220          final double percent = 100.0 * count / numSearches;
1221          out(p.getFirst().getName().toLowerCase(), " (",
1222               p.getFirst().intValue(), "):  ", p.getSecond(), " (",
1223               decimalFormat.format(percent), "%)");
1224        }
1225
1226        if (skippedWithSameCount.get() > 0L)
1227        {
1228          out("{ Skipped " + skippedWithSameCount.get() + " additional " +
1229               getSingularOrPlural(skippedWithSameCount.get(), "scope",
1230                    "scopes") +
1231               " with a count of " + count + " }");
1232        }
1233
1234        if (skippedWithLowerCount.get() > 0L)
1235        {
1236          out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
1237               getSingularOrPlural(skippedWithLowerCount.get(), "scope",
1238                    "scopes") +
1239               " with a count that is less than " + count + " }");
1240        }
1241      }
1242
1243      if (! searchEntryCounts.isEmpty())
1244      {
1245        final List<ObjectPair<Long,Long>> entryCounts = new ArrayList<>();
1246        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1247        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1248        getMostCommonElements(searchEntryCounts, entryCounts, displayCount,
1249             skippedWithSameCount, skippedWithLowerCount);
1250
1251        out();
1252        out("Most common search entry counts:");
1253
1254        long count = -1L;
1255        for (final ObjectPair<Long,Long> p : entryCounts)
1256        {
1257          count = p.getSecond();
1258          final double percent = 100.0 * count / numSearches;
1259          out(p.getFirst(), " matching ",
1260               getSingularOrPlural(p.getFirst(), "entry", "entries"),
1261               ":  ", p.getSecond(), " (", decimalFormat.format(percent), "%)");
1262        }
1263
1264        if (skippedWithSameCount.get() > 0L)
1265        {
1266          out("{ Skipped " + skippedWithSameCount.get() + " additional entry " +
1267               getSingularOrPlural(skippedWithSameCount.get(), "count",
1268                    "counts") +
1269               " with a count of " + count + " }");
1270        }
1271
1272        if (skippedWithLowerCount.get() > 0L)
1273        {
1274          out("{ Skipped " + skippedWithLowerCount.get() +
1275               " additional entry " +
1276               getSingularOrPlural(skippedWithLowerCount.get(), "count",
1277                    "counts") +
1278               " with a count that is less than " + count + " }");
1279        }
1280      }
1281
1282      printCounts(searchBaseDNs,
1283           "Most common base DNs for searches with a non-base scope:",
1284           "base DN", "base DNs");
1285
1286      printCounts(filterTypes,
1287           "Most common filters for searches with a non-base scope:",
1288           "filter", "filters");
1289
1290      if (numSearches > 0L)
1291      {
1292        long numSearchesMatchingNoEntries = 0L;
1293        for (final AtomicLong l : noEntryFilters.values())
1294        {
1295          numSearchesMatchingNoEntries += l.get();
1296        }
1297
1298        out();
1299        final double noEntryPercent =
1300             100.0 * numSearchesMatchingNoEntries / numSearches;
1301        out("Number of searches matching no entries:  ",
1302             numSearchesMatchingNoEntries, " (",
1303             decimalFormat.format(noEntryPercent), "%)");
1304
1305        printCounts(noEntryFilters,
1306             "Most common filters for searches matching no entries:",
1307             "filter", "filters");
1308
1309
1310        long numSearchesMatchingOneEntry = 0L;
1311        for (final AtomicLong l : oneEntryFilters.values())
1312        {
1313          numSearchesMatchingOneEntry += l.get();
1314        }
1315
1316        out();
1317        final double oneEntryPercent =
1318             100.0 * numSearchesMatchingOneEntry / numSearches;
1319        out("Number of searches matching one entry:  ",
1320             numSearchesMatchingOneEntry, " (",
1321             decimalFormat.format(oneEntryPercent), "%)");
1322
1323        printCounts(oneEntryFilters,
1324             "Most common filters for searches matching one entry:",
1325             "filter", "filters");
1326
1327
1328        long numSearchesMatchingMultipleEntries = 0L;
1329        for (final AtomicLong l : multiEntryFilters.values())
1330        {
1331          numSearchesMatchingMultipleEntries += l.get();
1332        }
1333
1334        out();
1335        final double multiEntryPercent =
1336             100.0 * numSearchesMatchingMultipleEntries / numSearches;
1337        out("Number of searches matching multiple entries:  ",
1338             numSearchesMatchingMultipleEntries, " (",
1339             decimalFormat.format(multiEntryPercent), "%)");
1340
1341        printCounts(multiEntryFilters,
1342             "Most common filters for searches matching multiple entries:",
1343             "filter", "filters");
1344      }
1345    }
1346
1347    if (! mostExpensiveFilters.isEmpty())
1348    {
1349        final List<ObjectPair<String,Long>> filterDurations = new ArrayList<>();
1350        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1351        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1352        getMostCommonElements(mostExpensiveFilters, filterDurations,
1353             displayCount, skippedWithSameCount, skippedWithLowerCount);
1354
1355        out();
1356        out("Filters for searches with the longest processing times:");
1357
1358        String durationStr = "";
1359        for (final ObjectPair<String,Long> p : filterDurations)
1360        {
1361          final long durationMicros = p.getSecond();
1362          final double durationMillis = durationMicros / 1_000.0;
1363          durationStr = decimalFormat.format(durationMillis) + " ms";
1364          out(p.getFirst(), ":  ", durationStr);
1365        }
1366
1367        if (skippedWithSameCount.get() > 0L)
1368        {
1369          out("{ Skipped " + skippedWithSameCount.get() + " additional " +
1370               getSingularOrPlural(skippedWithSameCount.get(), "filter",
1371                    "filters") +
1372               " with a duration of " + durationStr + " }");
1373        }
1374
1375        if (skippedWithLowerCount.get() > 0L)
1376        {
1377          out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
1378               getSingularOrPlural(skippedWithLowerCount.get(), "filter",
1379                    "filters") +
1380               " with a duration that is less than " + durationStr + " }");
1381        }
1382    }
1383
1384    final long totalUncached = numUncachedAdds + numUncachedBinds +
1385         numUncachedCompares + numUncachedDeletes + numUncachedExtended +
1386         numUncachedModifies + numUncachedModifyDNs + numUncachedSearches;
1387    if (totalUncached > 0L)
1388    {
1389      out();
1390      out("Operations accessing uncached data:");
1391      printUncached("Add", numUncachedAdds, numAdds);
1392      printUncached("Bind", numUncachedBinds, numBinds);
1393      printUncached("Compare", numUncachedCompares, numCompares);
1394      printUncached("Delete", numUncachedDeletes, numDeletes);
1395      printUncached("Extended", numUncachedExtended, numExtended);
1396      printUncached("Modify", numUncachedModifies, numModifies);
1397      printUncached("Modify DN", numUncachedModifyDNs, numModifyDNs);
1398      printUncached("Search", numUncachedSearches, numSearches);
1399    }
1400
1401
1402    return ResultCode.SUCCESS;
1403  }
1404
1405
1406
1407  /**
1408   * Retrieves a set of information that may be used to generate example usage
1409   * information.  Each element in the returned map should consist of a map
1410   * between an example set of arguments and a string that describes the
1411   * behavior of the tool when invoked with that set of arguments.
1412   *
1413   * @return  A set of information that may be used to generate example usage
1414   *          information.  It may be {@code null} or empty if no example usage
1415   *          information is available.
1416   */
1417  @Override()
1418  @NotNull()
1419  public LinkedHashMap<String[],String> getExampleUsages()
1420  {
1421    final LinkedHashMap<String[],String> examples =
1422         new LinkedHashMap<>(StaticUtils.computeMapCapacity(1));
1423
1424    final String[] args =
1425    {
1426      "/ds/logs/access"
1427    };
1428    final String description =
1429         "Analyze the contents of the /ds/logs/access access log file.";
1430    examples.put(args, description);
1431
1432    return examples;
1433  }
1434
1435
1436
1437  /**
1438   * Populates the provided processing time map with an initial set of values.
1439   *
1440   * @param  m  The processing time map to be populated.
1441   */
1442  private static void populateProcessingTimeMap(
1443                           @NotNull final HashMap<Long,AtomicLong> m)
1444  {
1445    m.put(1L, new AtomicLong(0L));
1446    m.put(2L, new AtomicLong(0L));
1447    m.put(3L, new AtomicLong(0L));
1448    m.put(5L, new AtomicLong(0L));
1449    m.put(10L, new AtomicLong(0L));
1450    m.put(20L, new AtomicLong(0L));
1451    m.put(30L, new AtomicLong(0L));
1452    m.put(50L, new AtomicLong(0L));
1453    m.put(100L, new AtomicLong(0L));
1454    m.put(1_000L, new AtomicLong(0L));
1455    m.put(2_000L, new AtomicLong(0L));
1456    m.put(3_000L, new AtomicLong(0L));
1457    m.put(5_000L, new AtomicLong(0L));
1458    m.put(10_000L, new AtomicLong(0L));
1459    m.put(20_000L, new AtomicLong(0L));
1460    m.put(30_000L, new AtomicLong(0L));
1461    m.put(60_000L, new AtomicLong(0L));
1462    m.put(Long.MAX_VALUE, new AtomicLong(0L));
1463  }
1464
1465
1466
1467  /**
1468   * Performs any necessary processing for a connect message.
1469   *
1470   * @param  m  The log message to be processed.
1471   */
1472  private void processConnect(@NotNull final ConnectAccessLogMessage m)
1473  {
1474    numConnects++;
1475
1476    final String clientAddr = m.getSourceAddress();
1477    if (clientAddr != null)
1478    {
1479      AtomicLong count = clientAddresses.get(clientAddr);
1480      if (count == null)
1481      {
1482        count = new AtomicLong(0L);
1483        clientAddresses.put(clientAddr, count);
1484      }
1485      count.incrementAndGet();
1486    }
1487
1488    final String ccp = m.getClientConnectionPolicy();
1489    if (ccp != null)
1490    {
1491      AtomicLong l = clientConnectionPolicies.get(ccp);
1492      if (l == null)
1493      {
1494        l = new AtomicLong(0L);
1495        clientConnectionPolicies.put(ccp, l);
1496      }
1497      l.incrementAndGet();
1498    }
1499  }
1500
1501
1502
1503  /**
1504   * Performs any necessary processing for a security negotiation message.
1505   *
1506   * @param  m  The log message to be processed.
1507   */
1508  private void processSecurityNegotiation(
1509                    @NotNull final SecurityNegotiationAccessLogMessage m)
1510  {
1511    final String protocol = m.getProtocol();
1512    if (protocol != null)
1513    {
1514      AtomicLong l = tlsProtocols.get(protocol);
1515      if (l == null)
1516      {
1517        l = new AtomicLong(0L);
1518        tlsProtocols.put(protocol, l);
1519      }
1520      l.incrementAndGet();
1521    }
1522
1523    final String cipherSuite = m.getCipher();
1524    if (cipherSuite != null)
1525    {
1526      AtomicLong l = tlsCipherSuites.get(cipherSuite);
1527      if (l == null)
1528      {
1529        l = new AtomicLong(0L);
1530        tlsCipherSuites.put(cipherSuite, l);
1531      }
1532      l.incrementAndGet();
1533    }
1534  }
1535
1536
1537
1538  /**
1539   * Performs any necessary processing for a disconnect message.
1540   *
1541   * @param  m  The log message to be processed.
1542   */
1543  private void processDisconnect(@NotNull final DisconnectAccessLogMessage m)
1544  {
1545    numDisconnects++;
1546
1547    final String reason = m.getDisconnectReason();
1548    if (reason != null)
1549    {
1550      AtomicLong l = disconnectReasons.get(reason);
1551      if (l == null)
1552      {
1553        l = new AtomicLong(0L);
1554        disconnectReasons.put(reason, l);
1555      }
1556      l.incrementAndGet();
1557    }
1558  }
1559
1560
1561
1562  /**
1563   * Performs any necessary processing for an abandon request message.
1564   *
1565   * @param  m  The log message to be processed.
1566   */
1567  private void processAbandonRequest(
1568                    @NotNull final AbandonRequestAccessLogMessage m)
1569  {
1570    numAbandons++;
1571  }
1572
1573
1574
1575  /**
1576   * Performs any necessary processing for an extended request message.
1577   *
1578   * @param  m  The log message to be processed.
1579   */
1580  private void processExtendedRequest(
1581                    @NotNull final ExtendedRequestAccessLogMessage m)
1582  {
1583    processedRequests.add(m.getConnectionID() + "-" + m.getOperationID());
1584    processExtendedRequestInternal(m);
1585  }
1586
1587
1588
1589  /**
1590   * Performs the internal processing for an extended request message.
1591   *
1592   * @param  m  The log message to be processed.
1593   */
1594  private void processExtendedRequestInternal(
1595                    @NotNull final ExtendedRequestAccessLogMessage m)
1596  {
1597    final String oid = m.getRequestOID();
1598    if (oid != null)
1599    {
1600      AtomicLong l = extendedOperations.get(oid);
1601      if (l == null)
1602      {
1603        l  = new AtomicLong(0L);
1604        extendedOperations.put(oid, l);
1605      }
1606      l.incrementAndGet();
1607
1608      final String requestType = m.getRequestType();
1609      if ((requestType != null) &&
1610           (! extendedOperationOIDsToNames.containsKey(oid)))
1611      {
1612        extendedOperationOIDsToNames.put(oid, requestType);
1613      }
1614    }
1615  }
1616
1617
1618
1619  /**
1620   * Performs any necessary processing for a search request message.
1621   *
1622   * @param  m  The log message to be processed.
1623   */
1624  private void processSearchRequest(
1625                    @NotNull final SearchRequestAccessLogMessage m)
1626  {
1627    processedRequests.add(m.getConnectionID() + "-" + m.getOperationID());
1628    processSearchRequestInternal(m);
1629  }
1630
1631
1632
1633  /**
1634   * Performs any necessary processing for a search request message.
1635   *
1636   * @param  m  The log message to be processed.
1637   */
1638  private void processSearchRequestInternal(
1639                    @NotNull final SearchRequestAccessLogMessage m)
1640  {
1641    final SearchScope scope = m.getScope();
1642    if (scope != null)
1643    {
1644      AtomicLong scopeCount = searchScopes.get(scope);
1645      if (scopeCount == null)
1646      {
1647        scopeCount = new AtomicLong(0L);
1648        searchScopes.put(scope, scopeCount);
1649      }
1650      scopeCount.incrementAndGet();
1651
1652      if (! scope.equals(SearchScope.BASE))
1653      {
1654        final String filterString = getFilterString(m.getParsedFilter());
1655        if (filterString != null)
1656        {
1657          AtomicLong filterCount = filterTypes.get(filterString);
1658          if (filterCount == null)
1659          {
1660            filterCount = new AtomicLong(0L);
1661            filterTypes.put(filterString, filterCount);
1662          }
1663          filterCount.incrementAndGet();
1664
1665
1666          final String baseDN = getDNString(m.getBaseDN());
1667          if (baseDN != null)
1668          {
1669            AtomicLong baseDNCount = searchBaseDNs.get(baseDN);
1670            if (baseDNCount == null)
1671            {
1672              baseDNCount = new AtomicLong(0L);
1673              searchBaseDNs.put(baseDN, baseDNCount);
1674            }
1675            baseDNCount.incrementAndGet();
1676          }
1677        }
1678      }
1679    }
1680  }
1681
1682
1683
1684  /**
1685   * Performs any necessary processing for an unbind request message.
1686   *
1687   * @param  m  The log message to be processed.
1688   */
1689  private void processUnbindRequest(
1690                    @NotNull final UnbindRequestAccessLogMessage m)
1691  {
1692    numUnbinds++;
1693  }
1694
1695
1696
1697  /**
1698   * Performs any necessary processing for an add result message.
1699   *
1700   * @param  m  The log message to be processed.
1701   */
1702  private void processAddResult(@NotNull final AddResultAccessLogMessage m)
1703  {
1704    numAdds++;
1705
1706    updateResultCodeCount(m.getResultCode(), addResultCodes);
1707    addProcessingDuration +=
1708         doubleValue(m.getProcessingTimeMillis(), addProcessingTimes);
1709
1710    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
1711    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
1712    {
1713      numUncachedAdds++;
1714    }
1715
1716    updateAuthzCount(m.getAlternateAuthorizationDN());
1717  }
1718
1719
1720
1721  /**
1722   * Performs any necessary processing for a bind result message.
1723   *
1724   * @param  m  The log message to be processed.
1725   */
1726  private void processBindResult(@NotNull final BindResultAccessLogMessage m)
1727  {
1728    numBinds++;
1729
1730    if (m.getAuthenticationType() != null)
1731    {
1732      final String authType;
1733      switch (m.getAuthenticationType())
1734      {
1735        case SIMPLE:
1736          authType = "Simple";
1737          break;
1738
1739        case SASL:
1740          final String saslMechanism = m.getSASLMechanismName();
1741          if (saslMechanism == null)
1742          {
1743            authType = "SASL {unknown mechanism}";
1744          }
1745          else
1746          {
1747            authType = "SASL " + saslMechanism;
1748          }
1749          break;
1750
1751        case INTERNAL:
1752          authType = "Internal";
1753          break;
1754
1755        default:
1756          authType = m.getAuthenticationType().name();
1757          break;
1758      }
1759
1760      AtomicLong l = authenticationTypes.get(authType);
1761      if (l == null)
1762      {
1763        l = new AtomicLong(0L);
1764        authenticationTypes.put(authType, l);
1765      }
1766      l.incrementAndGet();
1767    }
1768
1769    updateResultCodeCount(m.getResultCode(), bindResultCodes);
1770    bindProcessingDuration +=
1771         doubleValue(m.getProcessingTimeMillis(), bindProcessingTimes);
1772
1773    String authenticationDN = getDNString(m.getAuthenticationDN());
1774    if (m.getResultCode() == ResultCode.SUCCESS)
1775    {
1776      if (authenticationDN != null)
1777      {
1778        AtomicLong l = successfulBindDNs.get(authenticationDN);
1779        if (l == null)
1780        {
1781          l = new AtomicLong(0L);
1782          successfulBindDNs.put(authenticationDN, l);
1783        }
1784        l.incrementAndGet();
1785      }
1786
1787      final String ccp = m.getClientConnectionPolicy();
1788      if (ccp != null)
1789      {
1790        AtomicLong l = clientConnectionPolicies.get(ccp);
1791        if (l == null)
1792        {
1793          l = new AtomicLong(0L);
1794          clientConnectionPolicies.put(ccp, l);
1795        }
1796        l.incrementAndGet();
1797      }
1798    }
1799    else if ((m.getResultCode() != ResultCode.SASL_BIND_IN_PROGRESS) &&
1800         (m.getResultCode() != ResultCode.REFERRAL))
1801    {
1802      if (authenticationDN == null)
1803      {
1804        authenticationDN = getDNString(m.getDN());
1805      }
1806
1807      if (authenticationDN != null)
1808      {
1809        AtomicLong l = failedBindDNs.get(authenticationDN);
1810        if (l == null)
1811        {
1812          l = new AtomicLong(0L);
1813          failedBindDNs.put(authenticationDN, l);
1814        }
1815        l.incrementAndGet();
1816      }
1817    }
1818
1819    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
1820    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
1821    {
1822      numUncachedBinds++;
1823    }
1824
1825    updateAuthzCount(m.getAuthorizationDN());
1826  }
1827
1828
1829
1830  /**
1831   * Performs any necessary processing for a compare result message.
1832   *
1833   * @param  m  The log message to be processed.
1834   */
1835  private void processCompareResult(
1836                    @NotNull final CompareResultAccessLogMessage m)
1837  {
1838    numCompares++;
1839
1840    updateResultCodeCount(m.getResultCode(), compareResultCodes);
1841    compareProcessingDuration +=
1842         doubleValue(m.getProcessingTimeMillis(), compareProcessingTimes);
1843
1844    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
1845    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
1846    {
1847      numUncachedCompares++;
1848    }
1849
1850    updateAuthzCount(m.getAlternateAuthorizationDN());
1851  }
1852
1853
1854
1855  /**
1856   * Performs any necessary processing for a delete result message.
1857   *
1858   * @param  m  The log message to be processed.
1859   */
1860  private void processDeleteResult(
1861                    @NotNull final DeleteResultAccessLogMessage m)
1862  {
1863    numDeletes++;
1864
1865    updateResultCodeCount(m.getResultCode(), deleteResultCodes);
1866    deleteProcessingDuration +=
1867         doubleValue(m.getProcessingTimeMillis(), deleteProcessingTimes);
1868
1869    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
1870    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
1871    {
1872      numUncachedDeletes++;
1873    }
1874
1875    updateAuthzCount(m.getAlternateAuthorizationDN());
1876  }
1877
1878
1879
1880  /**
1881   * Performs any necessary processing for an extended result message.
1882   *
1883   * @param  m  The log message to be processed.
1884   */
1885  private void processExtendedResult(
1886                    @NotNull final ExtendedResultAccessLogMessage m)
1887  {
1888    numExtended++;
1889
1890    final String id = m.getConnectionID() + "-" + m.getOperationID();
1891    if (!processedRequests.remove(id))
1892    {
1893      processExtendedRequestInternal(m);
1894    }
1895
1896    updateResultCodeCount(m.getResultCode(), extendedResultCodes);
1897    extendedProcessingDuration +=
1898         doubleValue(m.getProcessingTimeMillis(), extendedProcessingTimes);
1899
1900    final String ccp = m.getClientConnectionPolicy();
1901    if (ccp != null)
1902    {
1903      AtomicLong l = clientConnectionPolicies.get(ccp);
1904      if (l == null)
1905      {
1906        l = new AtomicLong(0L);
1907        clientConnectionPolicies.put(ccp, l);
1908      }
1909      l.incrementAndGet();
1910    }
1911
1912    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
1913    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
1914    {
1915      numUncachedExtended++;
1916    }
1917  }
1918
1919
1920
1921  /**
1922   * Performs any necessary processing for a modify result message.
1923   *
1924   * @param  m  The log message to be processed.
1925   */
1926  private void processModifyResult(
1927                    @NotNull final ModifyResultAccessLogMessage m)
1928  {
1929    numModifies++;
1930
1931    updateResultCodeCount(m.getResultCode(), modifyResultCodes);
1932    modifyProcessingDuration +=
1933         doubleValue(m.getProcessingTimeMillis(), modifyProcessingTimes);
1934
1935    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
1936    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
1937    {
1938      numUncachedModifies++;
1939    }
1940
1941    updateAuthzCount(m.getAlternateAuthorizationDN());
1942  }
1943
1944
1945
1946  /**
1947   * Performs any necessary processing for a modify DN result message.
1948   *
1949   * @param  m  The log message to be processed.
1950   */
1951  private void processModifyDNResult(
1952                    @NotNull final ModifyDNResultAccessLogMessage m)
1953  {
1954    numModifyDNs++;
1955
1956    updateResultCodeCount(m.getResultCode(), modifyDNResultCodes);
1957    modifyDNProcessingDuration +=
1958         doubleValue(m.getProcessingTimeMillis(), modifyDNProcessingTimes);
1959
1960    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
1961    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
1962    {
1963      numUncachedModifyDNs++;
1964    }
1965
1966    updateAuthzCount(m.getAlternateAuthorizationDN());
1967  }
1968
1969
1970
1971  /**
1972   * Performs any necessary processing for a search result message.
1973   *
1974   * @param  m  The log message to be processed.
1975   */
1976  private void processSearchResult(
1977                    @NotNull final SearchResultAccessLogMessage m)
1978  {
1979    numSearches++;
1980
1981    final String id = m.getConnectionID() + "-" + m.getOperationID();
1982    if (! processedRequests.remove(id))
1983    {
1984      processSearchRequestInternal(m);
1985    }
1986
1987    final ResultCode resultCode = m.getResultCode();
1988    updateResultCodeCount(resultCode, searchResultCodes);
1989    searchProcessingDuration +=
1990         doubleValue(m.getProcessingTimeMillis(), searchProcessingTimes);
1991
1992    final String filterString = getFilterString(m.getParsedFilter());
1993
1994    final Long entryCount = m.getEntriesReturned();
1995    if (entryCount != null)
1996    {
1997      AtomicLong l = searchEntryCounts.get(entryCount);
1998      if (l == null)
1999      {
2000        l = new AtomicLong(0L);
2001        searchEntryCounts.put(entryCount, l);
2002      }
2003      l.incrementAndGet();
2004
2005      final Map<String,AtomicLong> filterCountMap;
2006      switch (entryCount.intValue())
2007      {
2008        case 0:
2009          filterCountMap = noEntryFilters;
2010          break;
2011        case 1:
2012          filterCountMap = oneEntryFilters;
2013          break;
2014        default:
2015          filterCountMap = multiEntryFilters;
2016          break;
2017      }
2018
2019      if (filterString != null)
2020      {
2021        AtomicLong filterCount = filterCountMap.get(filterString);
2022        if (filterCount == null)
2023        {
2024          filterCount = new AtomicLong(0L);
2025          filterCountMap.put(filterString, filterCount);
2026        }
2027        filterCount.incrementAndGet();
2028      }
2029    }
2030
2031    final Boolean isUnindexed = m.isUnindexed();
2032    if ((isUnindexed != null) && isUnindexed)
2033    {
2034      numUnindexedAttempts++;
2035      if (resultCode == ResultCode.SUCCESS)
2036      {
2037        numUnindexedSuccessful++;
2038      }
2039      else
2040      {
2041        numUnindexedFailed++;
2042      }
2043
2044      if (filterString != null)
2045      {
2046        AtomicLong l = unindexedFilters.get(filterString);
2047        if (l == null)
2048        {
2049          l = new AtomicLong(0L);
2050          unindexedFilters.put(filterString, l);
2051        }
2052        l.incrementAndGet();
2053      }
2054    }
2055
2056    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2057    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2058    {
2059      numUncachedSearches++;
2060    }
2061
2062    updateAuthzCount(m.getAlternateAuthorizationDN());
2063
2064    final Double processingTimeMillis = m.getProcessingTimeMillis();
2065    if ((processingTimeMillis != null) && (filterString != null))
2066    {
2067      final long processingTimeMicros =
2068           Math.round(processingTimeMillis * 1_000.0);
2069
2070      AtomicLong l = mostExpensiveFilters.get(filterString);
2071      if (l == null)
2072      {
2073        l = new AtomicLong(processingTimeMicros);
2074        mostExpensiveFilters.put(filterString, l);
2075      }
2076      else
2077      {
2078        final long previousProcessingTimeMicros = l.get();
2079        if (processingTimeMicros > previousProcessingTimeMicros)
2080        {
2081          l.set(processingTimeMicros);
2082        }
2083      }
2084    }
2085  }
2086
2087
2088
2089  /**
2090   * Updates the count for the provided result code in the given map.
2091   *
2092   * @param  rc  The result code for which to update the count.
2093   * @param  m   The map used to hold counts by result code.
2094   */
2095  private static void updateResultCodeCount(@Nullable final ResultCode rc,
2096                           @NotNull final HashMap<ResultCode,AtomicLong> m)
2097  {
2098    if (rc == null)
2099    {
2100      return;
2101    }
2102
2103    AtomicLong l = m.get(rc);
2104    if (l == null)
2105    {
2106      l = new AtomicLong(0L);
2107      m.put(rc, l);
2108    }
2109    l.incrementAndGet();
2110  }
2111
2112
2113
2114  /**
2115   * Retrieves the double value for the provided {@code Double} object.
2116   *
2117   * @param  d  The {@code Double} object for which to retrieve the value.
2118   * @param  m  The processing time histogram map to be updated.
2119   *
2120   * @return  The double value of the provided {@code Double} object if it was
2121   *          non-{@code null}, or 0.0 if it was {@code null}.
2122   */
2123  private static double doubleValue(@Nullable final Double d,
2124                                    @NotNull final HashMap<Long,AtomicLong> m)
2125  {
2126    if (d == null)
2127    {
2128      return 0.0;
2129    }
2130    else
2131    {
2132      for (final Map.Entry<Long,AtomicLong> e : m.entrySet())
2133      {
2134        if (d <= e.getKey())
2135        {
2136          e.getValue().incrementAndGet();
2137          break;
2138        }
2139      }
2140
2141      return d;
2142    }
2143  }
2144
2145
2146
2147  /**
2148   * Updates the provided list with the most frequently-occurring elements in
2149   * the provided map, paired with the number of times each value occurred.
2150   *
2151   * @param  <K>                    The type of object used as the key for the
2152   *                                provided map.
2153   * @param  countMap               The map to be examined.  It is expected that
2154   *                                the values of the map will be the count of
2155   *                                occurrences for the keys.
2156   * @param  mostCommonElementList  The list to which the values will be
2157   *                                updated.  It must not be {@code null}, must
2158   *                                be empty, and must be updatable.
2159   * @param  maxListSize            The maximum number of items to add to the
2160   *                                provided list.  It must be greater than
2161   *                                zero.
2162   * @param  skippedWithSameCount   A counter that will be incremented for each
2163   *                                map entry that is skipped with the same
2164   *                                count as a value that was not skipped.  It
2165   *                                must not be {@code null} and must initially
2166   *                                be zero.
2167   * @param  skippedWithLowerCount  A counter that will be incremented for each
2168   *                                map entry that is skipped with a lower count
2169   *                                as the last value that was not skipped.  It
2170   *                                must not be {@code null} and must initially
2171   *                                be zero.
2172   *
2173   * @return  A list of the most frequently-occurring elements in the provided
2174   *          map.
2175   */
2176  @NotNull()
2177  private static <K> List<ObjectPair<K,Long>> getMostCommonElements(
2178               @NotNull final Map<K,AtomicLong> countMap,
2179               @NotNull final List<ObjectPair<K,Long>> mostCommonElementList,
2180               final int maxListSize,
2181               @NotNull final AtomicLong skippedWithSameCount,
2182               @NotNull final AtomicLong skippedWithLowerCount)
2183  {
2184    final TreeMap<Long,List<K>> reverseMap =
2185         new TreeMap<>(new ReverseComparator<Long>());
2186    for (final Map.Entry<K,AtomicLong> e : countMap.entrySet())
2187    {
2188      final Long count = e.getValue().get();
2189      List<K> list = reverseMap.get(count);
2190      if (list == null)
2191      {
2192        list = new ArrayList<>();
2193        reverseMap.put(count, list);
2194      }
2195      list.add(e.getKey());
2196    }
2197
2198    for (final Map.Entry<Long,List<K>> e : reverseMap.entrySet())
2199    {
2200      final Long l = e.getKey();
2201      int numNotSkipped = 0;
2202      for (final K k : e.getValue())
2203      {
2204        if (mostCommonElementList.size() >= maxListSize)
2205        {
2206          if (numNotSkipped > 0)
2207          {
2208            skippedWithSameCount.incrementAndGet();
2209          }
2210          else
2211          {
2212            skippedWithLowerCount.incrementAndGet();
2213          }
2214        }
2215        else
2216        {
2217          numNotSkipped++;
2218          mostCommonElementList.add(new ObjectPair<>(k, l));
2219        }
2220      }
2221    }
2222
2223    return mostCommonElementList;
2224  }
2225
2226
2227
2228  /**
2229   * Updates the count of alternate authorization identities for the provided
2230   * DN.
2231   *
2232   * @param  authzDN  The DN of the alternate authorization identity that was
2233   *                  used.  It may be {@code null} if no alternate
2234   *                  authorization identity was used.
2235   */
2236  private void updateAuthzCount(@Nullable final String authzDN)
2237  {
2238    if (authzDN == null)
2239    {
2240      return;
2241    }
2242
2243    final String dnString = getDNString(authzDN);
2244
2245    AtomicLong l = authzDNs.get(dnString);
2246    if (l == null)
2247    {
2248      l = new AtomicLong(0L);
2249      authzDNs.put(dnString, l);
2250    }
2251  }
2252
2253
2254
2255  /**
2256   * Retrieves a string representation of the provided DN.  It may either be
2257   * anonymized, using question marks in place of specific attribute values, or
2258   * it may be the actual string representation of the given DN.
2259   *
2260   * @param  dn  The DN for which to retrieve the string representation.
2261   *
2262   * @return  A string representation of the provided DN, or {@code null} if the
2263   *          given DN was {@code null}.
2264   */
2265  @Nullable()
2266  private String getDNString(@Nullable final String dn)
2267  {
2268    if (dn == null)
2269    {
2270      return null;
2271    }
2272
2273    final DN parsedDN;
2274    try
2275    {
2276      parsedDN = new DN(dn);
2277    }
2278    catch (final Exception e)
2279    {
2280      Debug.debugException(e);
2281      return dn.toLowerCase();
2282    }
2283
2284    if (parsedDN.isNullDN())
2285    {
2286      return "{Null DN}";
2287    }
2288
2289    if (doNotAnonymize.isPresent())
2290    {
2291      return parsedDN.toNormalizedString();
2292    }
2293
2294    final StringBuilder buffer = new StringBuilder();
2295    final RDN[] rdns = parsedDN.getRDNs();
2296    for (int i=0; i < rdns.length; i++)
2297    {
2298      if (i > 0)
2299      {
2300        buffer.append(',');
2301      }
2302
2303      final RDN rdn = rdns[i];
2304      final String[] attributeNames = rdn.getAttributeNames();
2305      for (int j=0; j < attributeNames.length; j++)
2306      {
2307        if (j > 0)
2308        {
2309          buffer.append('+');
2310        }
2311        buffer.append(attributeNames[j].toLowerCase());
2312        buffer.append("=?");
2313      }
2314    }
2315
2316    return buffer.toString();
2317  }
2318
2319
2320
2321  /**
2322   * Retrieves a string representation of the provided filter.  It may
2323   * potentially be de-anonymized to include specific values.
2324   *
2325   * @param  filter  The filter for which to obtain the string representation.
2326   *
2327   * @return  A string representation of the provided filter (which may or may
2328   *          not be anonymized), or {@code null} if the provided filter is
2329   *          {@code null}.
2330   */
2331  @Nullable()
2332  private String getFilterString(@Nullable final Filter filter)
2333  {
2334    if (filter == null)
2335    {
2336      return null;
2337    }
2338
2339    if (doNotAnonymize.isPresent())
2340    {
2341      return filter.toString().toLowerCase();
2342    }
2343
2344    return new GenericFilter(filter).toString().toLowerCase();
2345  }
2346
2347
2348
2349  /**
2350   * Writes a breakdown of the processing times for a specified type of
2351   * operation.
2352   *
2353   * @param  t  The name of the operation type.
2354   * @param  n  The total number of operations of the specified type that were
2355   *            processed by the server.
2356   * @param  m  The map of operation counts by processing time bucket.
2357   */
2358  private void printProcessingTimeHistogram(@NotNull final String t,
2359                    final long n,
2360                    @NotNull final LinkedHashMap<Long,AtomicLong> m)
2361  {
2362    if (n <= 0)
2363    {
2364      return;
2365    }
2366
2367    out();
2368    out("Count of ", t, " operations by processing time:");
2369
2370    long lowerBound = 0;
2371    long accumulatedCount = 0;
2372    final Iterator<Map.Entry<Long,AtomicLong>> i = m.entrySet().iterator();
2373    while (i.hasNext())
2374    {
2375      final Map.Entry<Long,AtomicLong> e = i.next();
2376      final long upperBound = e.getKey();
2377      final long count = e.getValue().get();
2378      final double categoryPercent = 100.0 * count / n;
2379
2380      accumulatedCount += count;
2381      final double accumulatedPercent = 100.0 * accumulatedCount / n;
2382
2383      if (i.hasNext())
2384      {
2385        final String lowerBoundString;
2386        if (lowerBound == 0L)
2387        {
2388          lowerBoundString = "0 milliseconds";
2389        }
2390        else
2391        {
2392          final long lowerBoundNanos = lowerBound * 1_000_000L;
2393          lowerBoundString = DurationArgument.nanosToDuration(lowerBoundNanos);
2394        }
2395
2396        final long upperBoundNanos = upperBound * 1_000_000L;
2397        final String upperBoundString =
2398             DurationArgument.nanosToDuration(upperBoundNanos);
2399
2400
2401        out("Between ", lowerBoundString, " and ", upperBoundString, ":  ",
2402            count, " (", decimalFormat.format(categoryPercent), "%, ",
2403            decimalFormat.format(accumulatedPercent), "% accumulated)");
2404        lowerBound = upperBound;
2405      }
2406      else
2407      {
2408        final long lowerBoundNanos = lowerBound * 1_000_000L;
2409        final String lowerBoundString =
2410             DurationArgument.nanosToDuration(lowerBoundNanos);
2411
2412        out("Greater than ", lowerBoundString, ":  ", count, " (",
2413            decimalFormat.format(categoryPercent), "%, ",
2414            decimalFormat.format(accumulatedPercent), "% accumulated)");
2415      }
2416    }
2417  }
2418
2419
2420
2421  /**
2422   * Optionally prints information about the number and percent of operations of
2423   * the specified type that involved access to uncached data.
2424   *
2425   * @param  operationType  The type of operation.
2426   * @param  numUncached    The number of operations of the specified type that
2427   *                        involved access to uncached data.
2428   * @param  numTotal       The total number of operations of the specified
2429   *                        type.
2430   */
2431  private void printUncached(@NotNull final String operationType,
2432                             final long numUncached,
2433                             final long numTotal)
2434  {
2435    if (numUncached == 0)
2436    {
2437      return;
2438    }
2439
2440    out(operationType, ":  ", numUncached, " (",
2441         decimalFormat.format(100.0 * numUncached / numTotal), "%)");
2442  }
2443
2444
2445
2446  /**
2447   * Prints data from the provided map of counts.
2448   *
2449   * @param  countMap      The map containing the data to print.
2450   * @param  heading       The heading to display before printing the contents
2451   *                       of the map.
2452   * @param  singularItem  The name to use for a single item represented by the
2453   *                       key of the given map.
2454   * @param  pluralItem    The name to use for zero or multiple items
2455   *                       represented by the key of the given map.
2456   */
2457  private void printCounts(@Nullable final Map<String,AtomicLong> countMap,
2458                           @NotNull final String heading,
2459                           @NotNull final String singularItem,
2460                           @NotNull final String pluralItem)
2461  {
2462    if ((countMap == null) || countMap.isEmpty())
2463    {
2464      return;
2465    }
2466
2467    long totalCount = 0L;
2468    for (final AtomicLong l : countMap.values())
2469    {
2470      totalCount += l.get();
2471    }
2472
2473    out();
2474    out(heading);
2475
2476    int displayCount = reportCount.getValue();
2477    if (displayCount <= 0L)
2478    {
2479      displayCount = Integer.MAX_VALUE;
2480    }
2481
2482    final List<ObjectPair<String,Long>> countList = new ArrayList<>();
2483    final AtomicLong skippedWithSameCount = new AtomicLong(0L);
2484    final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
2485    getMostCommonElements(countMap, countList, displayCount,
2486         skippedWithSameCount, skippedWithLowerCount);
2487
2488    long count = -1L;
2489    for (final ObjectPair<String,Long> p : countList)
2490    {
2491      count = p.getSecond();
2492
2493      if (totalCount > 0L)
2494      {
2495        final double percent = 100.0 * count / totalCount;
2496        out(p.getFirst(), ":  ", count, " (", decimalFormat.format(percent),
2497             ")");
2498      }
2499      else
2500      {
2501        out(p.getFirst(), ":  ", count);
2502      }
2503    }
2504
2505    if (skippedWithSameCount.get() > 0L)
2506    {
2507      out("{ Skipped " + skippedWithSameCount.get() + " additional " +
2508           getSingularOrPlural(skippedWithSameCount.get(), singularItem,
2509                pluralItem) +
2510           " with a count of " + count + " }");
2511    }
2512
2513    if (skippedWithLowerCount.get() > 0L)
2514    {
2515      out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
2516           getSingularOrPlural(skippedWithLowerCount.get(), singularItem,
2517                pluralItem) +
2518           " with a count that is less than " + count + " }");
2519    }
2520  }
2521
2522
2523
2524  /**
2525   * Prints data from the provided map of counts.
2526   *
2527   * @param  countMap       The map containing the data to print.
2528   * @param  operationType  The type of operation represented by the keys of
2529   *                        the map.
2530   */
2531  private void printResultCodeCounts(
2532                    @Nullable final Map<ResultCode,AtomicLong> countMap,
2533                    @NotNull final String operationType)
2534  {
2535    if ((countMap == null) || countMap.isEmpty())
2536    {
2537      return;
2538    }
2539
2540    long totalCount = 0L;
2541    for (final AtomicLong l : countMap.values())
2542    {
2543      totalCount += l.get();
2544    }
2545
2546    out();
2547    out("Most common " + operationType + " operation result codes:");
2548
2549    int displayCount = reportCount.getValue();
2550    if (displayCount <= 0L)
2551    {
2552      displayCount = Integer.MAX_VALUE;
2553    }
2554
2555    final List<ObjectPair<ResultCode,Long>> resultCodeList = new ArrayList<>();
2556    final AtomicLong skippedWithSameCount = new AtomicLong(0L);
2557    final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
2558    getMostCommonElements(countMap, resultCodeList, displayCount,
2559         skippedWithSameCount, skippedWithLowerCount);
2560
2561    long count = -1L;
2562    for (final ObjectPair<ResultCode,Long> p : resultCodeList)
2563    {
2564      count = p.getSecond();
2565
2566      if (totalCount > 0L)
2567      {
2568        final double percent = 100.0 * count / totalCount;
2569        out(p.getFirst().getName(), " (", p.getFirst().intValue(), "):  ",
2570             count, " (", decimalFormat.format(percent), ")");
2571      }
2572      else
2573      {
2574        out(p.getFirst(), ":  ", count);
2575      }
2576    }
2577
2578    if (skippedWithSameCount.get() > 0L)
2579    {
2580      out("{ Skipped " + skippedWithSameCount.get() + " additional result " +
2581           getSingularOrPlural(skippedWithSameCount.get(), "code", "codes") +
2582           " with a count of " + count + " }");
2583    }
2584
2585    if (skippedWithLowerCount.get() > 0L)
2586    {
2587      out("{ Skipped " + skippedWithLowerCount.get() + " additional result " +
2588           getSingularOrPlural(skippedWithLowerCount.get(), "code", "codes") +
2589           " with a count that is less than " + count + " }");
2590    }
2591  }
2592
2593
2594
2595  /**
2596   * Retrieves the appropriate singular or plural form based on the given
2597   * value.
2598   *
2599   * @param  count     The count that will be used to determine whether to
2600   *                   retrieve the singular or plural form.
2601   * @param  singular  The singular form for the value to return.
2602   * @param  plural    The plural form for the value to return.
2603   *
2604   * @return  The singular form if the count is 1, or the plural form if the
2605   *          count is any other value.
2606   */
2607  @NotNull()
2608  private String getSingularOrPlural(final long count,
2609                                     @NotNull final String singular,
2610                                     @NotNull final String plural)
2611  {
2612    if (count == 1L)
2613    {
2614      return singular;
2615    }
2616    else
2617    {
2618      return plural;
2619    }
2620  }
2621}