001/* 002 * Copyright 2009-2021 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2009-2021 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2009-2021 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.unboundidds.examples; 037 038 039 040import java.io.File; 041import java.io.FileInputStream; 042import java.io.InputStream; 043import java.io.InputStreamReader; 044import java.io.IOException; 045import java.io.OutputStream; 046import java.io.Serializable; 047import java.text.DecimalFormat; 048import java.util.ArrayList; 049import java.util.HashMap; 050import java.util.HashSet; 051import java.util.Iterator; 052import java.util.LinkedHashMap; 053import java.util.List; 054import java.util.Map; 055import java.util.TreeMap; 056import java.util.concurrent.atomic.AtomicLong; 057import java.util.zip.GZIPInputStream; 058import javax.crypto.BadPaddingException; 059 060import com.unboundid.ldap.sdk.DN; 061import com.unboundid.ldap.sdk.Filter; 062import com.unboundid.ldap.sdk.LDAPException; 063import com.unboundid.ldap.sdk.RDN; 064import com.unboundid.ldap.sdk.ResultCode; 065import com.unboundid.ldap.sdk.SearchScope; 066import com.unboundid.ldap.sdk.Version; 067import com.unboundid.ldap.sdk.unboundidds.logs.AbandonRequestAccessLogMessage; 068import com.unboundid.ldap.sdk.unboundidds.logs.AccessLogMessage; 069import com.unboundid.ldap.sdk.unboundidds.logs.AccessLogReader; 070import com.unboundid.ldap.sdk.unboundidds.logs.AddResultAccessLogMessage; 071import com.unboundid.ldap.sdk.unboundidds.logs.BindResultAccessLogMessage; 072import com.unboundid.ldap.sdk.unboundidds.logs.CompareResultAccessLogMessage; 073import com.unboundid.ldap.sdk.unboundidds.logs.ConnectAccessLogMessage; 074import com.unboundid.ldap.sdk.unboundidds.logs.DeleteResultAccessLogMessage; 075import com.unboundid.ldap.sdk.unboundidds.logs.DisconnectAccessLogMessage; 076import com.unboundid.ldap.sdk.unboundidds.logs.ExtendedRequestAccessLogMessage; 077import com.unboundid.ldap.sdk.unboundidds.logs.ExtendedResultAccessLogMessage; 078import com.unboundid.ldap.sdk.unboundidds.logs.LogException; 079import com.unboundid.ldap.sdk.unboundidds.logs.ModifyDNResultAccessLogMessage; 080import com.unboundid.ldap.sdk.unboundidds.logs.ModifyResultAccessLogMessage; 081import com.unboundid.ldap.sdk.unboundidds.logs.OperationAccessLogMessage; 082import com.unboundid.ldap.sdk.unboundidds.logs.SearchRequestAccessLogMessage; 083import com.unboundid.ldap.sdk.unboundidds.logs.SearchResultAccessLogMessage; 084import com.unboundid.ldap.sdk.unboundidds.logs. 085 SecurityNegotiationAccessLogMessage; 086import com.unboundid.ldap.sdk.unboundidds.logs.UnbindRequestAccessLogMessage; 087import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 088import com.unboundid.util.CommandLineTool; 089import com.unboundid.util.Debug; 090import com.unboundid.util.NotMutable; 091import com.unboundid.util.NotNull; 092import com.unboundid.util.Nullable; 093import com.unboundid.util.ObjectPair; 094import com.unboundid.util.ReverseComparator; 095import com.unboundid.util.StaticUtils; 096import com.unboundid.util.ThreadSafety; 097import com.unboundid.util.ThreadSafetyLevel; 098import com.unboundid.util.args.ArgumentException; 099import com.unboundid.util.args.ArgumentParser; 100import com.unboundid.util.args.BooleanArgument; 101import com.unboundid.util.args.DurationArgument; 102import com.unboundid.util.args.FileArgument; 103import com.unboundid.util.args.IntegerArgument; 104 105 106 107/** 108 * This class provides a tool that may be used to read and summarize the 109 * contents of one or more access log files from Ping Identity, UnboundID and 110 * Nokia/Alcatel-Lucent 8661 server products. 111 * <BR> 112 * <BLOCKQUOTE> 113 * <B>NOTE:</B> This class, and other classes within the 114 * {@code com.unboundid.ldap.sdk.unboundidds} package structure, are only 115 * supported for use against Ping Identity, UnboundID, and 116 * Nokia/Alcatel-Lucent 8661 server products. These classes provide support 117 * for proprietary functionality or for external specifications that are not 118 * considered stable or mature enough to be guaranteed to work in an 119 * interoperable way with other types of LDAP servers. 120 * </BLOCKQUOTE> 121 * Information that will be reported includes: 122 * <UL> 123 * <LI>The total length of time covered by the log files.</LI> 124 * <LI>The number of connections established and disconnected, the addresses 125 * of the most commonly-connecting clients, and the average rate of 126 * connects and disconnects per second.</LI> 127 * <LI>The number of operations processed, overall and by operation type, 128 * and the average rate of operations per second.</LI> 129 * <LI>The average duration for operations processed, overall and by operation 130 * type.</LI> 131 * <LI>A breakdown of operation processing times into a number of predefined 132 * categories, ranging from less than one millisecond to over one 133 * minute.</LI> 134 * <LI>A breakdown of the most common result codes for each type of operation 135 * and their relative frequencies.</LI> 136 * <LI>The most common types of extended requests processed and their 137 * relative frequencies.</LI> 138 * <LI>The number of unindexed search operations processed and the most common 139 * types of filters used in unindexed searches.</LI> 140 * <LI>A breakdown of the relative frequencies for each type of search 141 * scope.</LI> 142 * <LI>The most common types of search filters used for search 143 * operations and their relative frequencies.</LI> 144 * </UL> 145 * It is designed to work with access log files using either the default log 146 * format with separate request and response messages, as well as log files 147 * in which the request and response details have been combined on the same 148 * line. The log files to be processed should be provided as command-line 149 * arguments. 150 * <BR><BR> 151 * The APIs demonstrated by this example include: 152 * <UL> 153 * <LI>Access log parsing (from the 154 * {@code com.unboundid.ldap.sdk.unboundidds.logs} package)</LI> 155 * <LI>Argument parsing (from the {@code com.unboundid.util.args} 156 * package)</LI> 157 * </UL> 158 */ 159@NotMutable() 160@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 161public final class SummarizeAccessLog 162 extends CommandLineTool 163 implements Serializable 164{ 165 /** 166 * The serial version UID for this serializable class. 167 */ 168 private static final long serialVersionUID = 7189168366509887130L; 169 170 171 172 // Variables used for accessing argument information. 173 @Nullable private ArgumentParser argumentParser; 174 175 // An argument that may be used to indicate that the summarized output should 176 // not be anonymized, and should include attribute values. 177 @Nullable private BooleanArgument doNotAnonymize; 178 179 // An argument that may be used to indicate that the log files are compressed. 180 @Nullable private BooleanArgument isCompressed; 181 182 // An argument used to specify the encryption passphrase. 183 @Nullable private FileArgument encryptionPassphraseFile; 184 185 // An argument used to specify the maximum number of values to report for each 186 // item. 187 @Nullable private IntegerArgument reportCount; 188 189 // The decimal format that will be used for this class. 190 @NotNull private final DecimalFormat decimalFormat; 191 192 // The total duration for log content, in milliseconds. 193 private long logDurationMillis; 194 195 // The total processing time for each type of operation. 196 private double addProcessingDuration; 197 private double bindProcessingDuration; 198 private double compareProcessingDuration; 199 private double deleteProcessingDuration; 200 private double extendedProcessingDuration; 201 private double modifyProcessingDuration; 202 private double modifyDNProcessingDuration; 203 private double searchProcessingDuration; 204 205 // A variable used for counting the number of messages of each type. 206 private long numAbandons; 207 private long numAdds; 208 private long numBinds; 209 private long numCompares; 210 private long numConnects; 211 private long numDeletes; 212 private long numDisconnects; 213 private long numExtended; 214 private long numModifies; 215 private long numModifyDNs; 216 private long numSearches; 217 private long numUnbinds; 218 219 // The number of operations of each type that accessed uncached data. 220 private long numUncachedAdds; 221 private long numUncachedBinds; 222 private long numUncachedCompares; 223 private long numUncachedDeletes; 224 private long numUncachedExtended; 225 private long numUncachedModifies; 226 private long numUncachedModifyDNs; 227 private long numUncachedSearches; 228 229 // The number of unindexed searches processed within the server. 230 private long numUnindexedAttempts; 231 private long numUnindexedFailed; 232 private long numUnindexedSuccessful; 233 234 // Variables used for maintaining counts for common types of information. 235 @NotNull private final HashMap<Long,AtomicLong> searchEntryCounts; 236 @NotNull private final HashMap<ResultCode,AtomicLong> addResultCodes; 237 @NotNull private final HashMap<ResultCode,AtomicLong> bindResultCodes; 238 @NotNull private final HashMap<ResultCode,AtomicLong> compareResultCodes; 239 @NotNull private final HashMap<ResultCode,AtomicLong> deleteResultCodes; 240 @NotNull private final HashMap<ResultCode,AtomicLong> extendedResultCodes; 241 @NotNull private final HashMap<ResultCode,AtomicLong> modifyResultCodes; 242 @NotNull private final HashMap<ResultCode,AtomicLong> modifyDNResultCodes; 243 @NotNull private final HashMap<ResultCode,AtomicLong> searchResultCodes; 244 @NotNull private final HashMap<SearchScope,AtomicLong> searchScopes; 245 @NotNull private final HashMap<String,AtomicLong> authenticationTypes; 246 @NotNull private final HashMap<String,AtomicLong> authzDNs; 247 @NotNull private final HashMap<String,AtomicLong> failedBindDNs; 248 @NotNull private final HashMap<String,AtomicLong> successfulBindDNs; 249 @NotNull private final HashMap<String,AtomicLong> clientAddresses; 250 @NotNull private final HashMap<String,AtomicLong> clientConnectionPolicies; 251 @NotNull private final HashMap<String,AtomicLong> disconnectReasons; 252 @NotNull private final HashMap<String,AtomicLong> extendedOperations; 253 @NotNull private final HashMap<String,AtomicLong> filterTypes; 254 @NotNull private final HashMap<String,AtomicLong> mostExpensiveFilters; 255 @NotNull private final HashMap<String,AtomicLong> multiEntryFilters; 256 @NotNull private final HashMap<String,AtomicLong> noEntryFilters; 257 @NotNull private final HashMap<String,AtomicLong> oneEntryFilters; 258 @NotNull private final HashMap<String,AtomicLong> searchBaseDNs; 259 @NotNull private final HashMap<String,AtomicLong> tlsCipherSuites; 260 @NotNull private final HashMap<String,AtomicLong> tlsProtocols; 261 @NotNull private final HashMap<String,AtomicLong> unindexedFilters; 262 @NotNull private final HashMap<String,String> extendedOperationOIDsToNames; 263 @NotNull private final HashSet<String> processedRequests; 264 @NotNull private final LinkedHashMap<Long,AtomicLong> addProcessingTimes; 265 @NotNull private final LinkedHashMap<Long,AtomicLong> bindProcessingTimes; 266 @NotNull private final LinkedHashMap<Long,AtomicLong> compareProcessingTimes; 267 @NotNull private final LinkedHashMap<Long,AtomicLong> deleteProcessingTimes; 268 @NotNull private final LinkedHashMap<Long,AtomicLong> extendedProcessingTimes; 269 @NotNull private final LinkedHashMap<Long,AtomicLong> modifyProcessingTimes; 270 @NotNull private final LinkedHashMap<Long,AtomicLong> modifyDNProcessingTimes; 271 @NotNull private final LinkedHashMap<Long,AtomicLong> searchProcessingTimes; 272 273 274 275 /** 276 * Parse the provided command line arguments and perform the appropriate 277 * processing. 278 * 279 * @param args The command line arguments provided to this program. 280 */ 281 public static void main(@NotNull final String[] args) 282 { 283 final ResultCode resultCode = main(args, System.out, System.err); 284 if (resultCode != ResultCode.SUCCESS) 285 { 286 System.exit(resultCode.intValue()); 287 } 288 } 289 290 291 292 /** 293 * Parse the provided command line arguments and perform the appropriate 294 * processing. 295 * 296 * @param args The command line arguments provided to this program. 297 * @param outStream The output stream to which standard out should be 298 * written. It may be {@code null} if output should be 299 * suppressed. 300 * @param errStream The output stream to which standard error should be 301 * written. It may be {@code null} if error messages 302 * should be suppressed. 303 * 304 * @return A result code indicating whether the processing was successful. 305 */ 306 @NotNull() 307 public static ResultCode main(@NotNull final String[] args, 308 @Nullable final OutputStream outStream, 309 @Nullable final OutputStream errStream) 310 { 311 final SummarizeAccessLog summarizer = 312 new SummarizeAccessLog(outStream, errStream); 313 return summarizer.runTool(args); 314 } 315 316 317 318 /** 319 * Creates a new instance of this tool. 320 * 321 * @param outStream The output stream to which standard out should be 322 * written. It may be {@code null} if output should be 323 * suppressed. 324 * @param errStream The output stream to which standard error should be 325 * written. It may be {@code null} if error messages 326 * should be suppressed. 327 */ 328 public SummarizeAccessLog(@Nullable final OutputStream outStream, 329 @Nullable final OutputStream errStream) 330 { 331 super(outStream, errStream); 332 333 argumentParser = null; 334 doNotAnonymize = null; 335 isCompressed = null; 336 encryptionPassphraseFile = null; 337 reportCount = null; 338 339 decimalFormat = new DecimalFormat("0.000"); 340 341 logDurationMillis = 0L; 342 343 addProcessingDuration = 0.0; 344 bindProcessingDuration = 0.0; 345 compareProcessingDuration = 0.0; 346 deleteProcessingDuration = 0.0; 347 extendedProcessingDuration = 0.0; 348 modifyProcessingDuration = 0.0; 349 modifyDNProcessingDuration = 0.0; 350 searchProcessingDuration = 0.0; 351 352 numAbandons = 0L; 353 numAdds = 0L; 354 numBinds = 0L; 355 numCompares = 0L; 356 numConnects = 0L; 357 numDeletes = 0L; 358 numDisconnects = 0L; 359 numExtended = 0L; 360 numModifies = 0L; 361 numModifyDNs = 0L; 362 numSearches = 0L; 363 numUnbinds = 0L; 364 365 numUncachedAdds = 0L; 366 numUncachedBinds = 0L; 367 numUncachedCompares = 0L; 368 numUncachedDeletes = 0L; 369 numUncachedExtended = 0L; 370 numUncachedModifies = 0L; 371 numUncachedModifyDNs = 0L; 372 numUncachedSearches = 0L; 373 374 numUnindexedAttempts = 0L; 375 numUnindexedFailed = 0L; 376 numUnindexedSuccessful = 0L; 377 378 searchEntryCounts = new HashMap<>(StaticUtils.computeMapCapacity(10)); 379 addResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 380 bindResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 381 compareResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 382 deleteResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 383 extendedResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 384 modifyResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 385 modifyDNResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 386 searchResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 387 searchScopes = new HashMap<>(StaticUtils.computeMapCapacity(4)); 388 authenticationTypes = new HashMap<>(StaticUtils.computeMapCapacity(100)); 389 authzDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 390 failedBindDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 391 successfulBindDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 392 clientAddresses = new HashMap<>(StaticUtils.computeMapCapacity(100)); 393 clientConnectionPolicies = 394 new HashMap<>(StaticUtils.computeMapCapacity(100)); 395 disconnectReasons = new HashMap<>(StaticUtils.computeMapCapacity(100)); 396 extendedOperations = new HashMap<>(StaticUtils.computeMapCapacity(10)); 397 filterTypes = new HashMap<>(StaticUtils.computeMapCapacity(100)); 398 mostExpensiveFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 399 multiEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 400 noEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 401 oneEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 402 searchBaseDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 403 tlsCipherSuites = new HashMap<>(StaticUtils.computeMapCapacity(100)); 404 tlsProtocols = new HashMap<>(StaticUtils.computeMapCapacity(100)); 405 unindexedFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 406 extendedOperationOIDsToNames = 407 new HashMap<>(StaticUtils.computeMapCapacity(100)); 408 processedRequests = new HashSet<>(StaticUtils.computeMapCapacity(100)); 409 addProcessingTimes = 410 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 411 bindProcessingTimes = 412 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 413 compareProcessingTimes = 414 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 415 deleteProcessingTimes = 416 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 417 extendedProcessingTimes = 418 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 419 modifyProcessingTimes = 420 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 421 modifyDNProcessingTimes = 422 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 423 searchProcessingTimes = 424 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 425 426 populateProcessingTimeMap(addProcessingTimes); 427 populateProcessingTimeMap(bindProcessingTimes); 428 populateProcessingTimeMap(compareProcessingTimes); 429 populateProcessingTimeMap(deleteProcessingTimes); 430 populateProcessingTimeMap(extendedProcessingTimes); 431 populateProcessingTimeMap(modifyProcessingTimes); 432 populateProcessingTimeMap(modifyDNProcessingTimes); 433 populateProcessingTimeMap(searchProcessingTimes); 434 } 435 436 437 438 /** 439 * Retrieves the name for this tool. 440 * 441 * @return The name for this tool. 442 */ 443 @Override() 444 @NotNull() 445 public String getToolName() 446 { 447 return "summarize-access-log"; 448 } 449 450 451 452 /** 453 * Retrieves the description for this tool. 454 * 455 * @return The description for this tool. 456 */ 457 @Override() 458 @NotNull() 459 public String getToolDescription() 460 { 461 return "Examine one or more access log files from Ping Identity, " + 462 "UnboundID, or Nokia/Alcatel-Lucent 8661 server products to display " + 463 "a number of metrics about operations processed within the server."; 464 } 465 466 467 468 /** 469 * Retrieves the version string for this tool. 470 * 471 * @return The version string for this tool. 472 */ 473 @Override() 474 @NotNull() 475 public String getToolVersion() 476 { 477 return Version.NUMERIC_VERSION_STRING; 478 } 479 480 481 482 /** 483 * Retrieves the minimum number of unnamed trailing arguments that are 484 * required. 485 * 486 * @return One, to indicate that at least one trailing argument (representing 487 * the path to an access log file) must be provided. 488 */ 489 @Override() 490 public int getMinTrailingArguments() 491 { 492 return 1; 493 } 494 495 496 497 /** 498 * Retrieves the maximum number of unnamed trailing arguments that may be 499 * provided for this tool. 500 * 501 * @return The maximum number of unnamed trailing arguments that may be 502 * provided for this tool. 503 */ 504 @Override() 505 public int getMaxTrailingArguments() 506 { 507 return -1; 508 } 509 510 511 512 /** 513 * Retrieves a placeholder string that should be used for trailing arguments 514 * in the usage information for this tool. 515 * 516 * @return A placeholder string that should be used for trailing arguments in 517 * the usage information for this tool. 518 */ 519 @Override() 520 @NotNull() 521 public String getTrailingArgumentsPlaceholder() 522 { 523 return "{path}"; 524 } 525 526 527 528 /** 529 * Indicates whether this tool should provide support for an interactive mode, 530 * in which the tool offers a mode in which the arguments can be provided in 531 * a text-driven menu rather than requiring them to be given on the command 532 * line. If interactive mode is supported, it may be invoked using the 533 * "--interactive" argument. Alternately, if interactive mode is supported 534 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 535 * interactive mode may be invoked by simply launching the tool without any 536 * arguments. 537 * 538 * @return {@code true} if this tool supports interactive mode, or 539 * {@code false} if not. 540 */ 541 @Override() 542 public boolean supportsInteractiveMode() 543 { 544 return true; 545 } 546 547 548 549 /** 550 * Indicates whether this tool defaults to launching in interactive mode if 551 * the tool is invoked without any command-line arguments. This will only be 552 * used if {@link #supportsInteractiveMode()} returns {@code true}. 553 * 554 * @return {@code true} if this tool defaults to using interactive mode if 555 * launched without any command-line arguments, or {@code false} if 556 * not. 557 */ 558 @Override() 559 public boolean defaultsToInteractiveMode() 560 { 561 return true; 562 } 563 564 565 566 /** 567 * Indicates whether this tool should provide arguments for redirecting output 568 * to a file. If this method returns {@code true}, then the tool will offer 569 * an "--outputFile" argument that will specify the path to a file to which 570 * all standard output and standard error content will be written, and it will 571 * also offer a "--teeToStandardOut" argument that can only be used if the 572 * "--outputFile" argument is present and will cause all output to be written 573 * to both the specified output file and to standard output. 574 * 575 * @return {@code true} if this tool should provide arguments for redirecting 576 * output to a file, or {@code false} if not. 577 */ 578 @Override() 579 protected boolean supportsOutputFile() 580 { 581 return true; 582 } 583 584 585 586 /** 587 * Indicates whether this tool supports the use of a properties file for 588 * specifying default values for arguments that aren't specified on the 589 * command line. 590 * 591 * @return {@code true} if this tool supports the use of a properties file 592 * for specifying default values for arguments that aren't specified 593 * on the command line, or {@code false} if not. 594 */ 595 @Override() 596 public boolean supportsPropertiesFile() 597 { 598 return true; 599 } 600 601 602 603 /** 604 * Adds the command-line arguments supported for use with this tool to the 605 * provided argument parser. The tool may need to retain references to the 606 * arguments (and/or the argument parser, if trailing arguments are allowed) 607 * to it in order to obtain their values for use in later processing. 608 * 609 * @param parser The argument parser to which the arguments are to be added. 610 * 611 * @throws ArgumentException If a problem occurs while adding any of the 612 * tool-specific arguments to the provided 613 * argument parser. 614 */ 615 @Override() 616 public void addToolArguments(@NotNull final ArgumentParser parser) 617 throws ArgumentException 618 { 619 // We need to save a reference to the argument parser so that we can get 620 // the trailing arguments later. 621 argumentParser = parser; 622 623 // Add an argument that makes it possible to read a compressed log file. 624 // Note that this argument is no longer needed for dealing with compressed 625 // files, since the tool will automatically detect whether a file is 626 // compressed. However, the argument is still provided for the purpose of 627 // backward compatibility. 628 String description = "Indicates that the log file is compressed."; 629 isCompressed = new BooleanArgument('c', "isCompressed", description); 630 isCompressed.addLongIdentifier("is-compressed", true); 631 isCompressed.addLongIdentifier("compressed", true); 632 isCompressed.setHidden(true); 633 parser.addArgument(isCompressed); 634 635 636 // Add an argument that indicates that the tool should read the encryption 637 // passphrase from a file. 638 description = "Indicates that the log file is encrypted and that the " + 639 "encryption passphrase is contained in the specified file. If " + 640 "the log data is encrypted and this argument is not provided, then " + 641 "the tool will interactively prompt for the encryption passphrase."; 642 encryptionPassphraseFile = new FileArgument(null, 643 "encryptionPassphraseFile", false, 1, null, description, true, true, 644 true, false); 645 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 646 true); 647 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 648 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 649 true); 650 parser.addArgument(encryptionPassphraseFile); 651 652 653 // Add an argument that indicates the number of values to display for each 654 // item being summarized. 655 description = "The number of values to display for each item being " + 656 "summarized. A value of zero indicates that all items should be " + 657 "displayed. If this is not provided, a default value of 20 will " + 658 "be used."; 659 reportCount = new IntegerArgument(null, "reportCount", false, 0, null, 660 description, 0, Integer.MAX_VALUE, 20); 661 reportCount.addLongIdentifier("report-count", true); 662 reportCount.addLongIdentifier("maximumCount", true); 663 reportCount.addLongIdentifier("maximum-count", true); 664 reportCount.addLongIdentifier("maxCount", true); 665 reportCount.addLongIdentifier("max-count", true); 666 reportCount.addLongIdentifier("count", true); 667 parser.addArgument(reportCount); 668 669 670 // Add an argument that indicates that the output should not be anonymized. 671 description = "Do not anonymize the output, but include actual attribute " + 672 "values in filters and DNs. This will also have the effect of " + 673 "de-generifying those values, so output including the most common " + 674 "filters and DNs in some category will be specific instances of " + 675 "those filters and DNs instead of generic patterns."; 676 doNotAnonymize = new BooleanArgument(null, "doNotAnonymize", 1, 677 description); 678 doNotAnonymize.addLongIdentifier("do-not-anonymize", true); 679 doNotAnonymize.addLongIdentifier("deAnonymize", true); 680 doNotAnonymize.addLongIdentifier("de-anonymize", true); 681 parser.addArgument(doNotAnonymize); 682 } 683 684 685 686 /** 687 * Performs any necessary processing that should be done to ensure that the 688 * provided set of command-line arguments were valid. This method will be 689 * called after the basic argument parsing has been performed and immediately 690 * before the {@link #doToolProcessing} method is invoked. 691 * 692 * @throws ArgumentException If there was a problem with the command-line 693 * arguments provided to this program. 694 */ 695 @Override() 696 public void doExtendedArgumentValidation() 697 throws ArgumentException 698 { 699 // Make sure that at least one access log file path was provided. 700 final List<String> trailingArguments = 701 argumentParser.getTrailingArguments(); 702 if ((trailingArguments == null) || trailingArguments.isEmpty()) 703 { 704 throw new ArgumentException("No access log file paths were provided."); 705 } 706 } 707 708 709 710 /** 711 * Performs the core set of processing for this tool. 712 * 713 * @return A result code that indicates whether the processing completed 714 * successfully. 715 */ 716 @Override() 717 @NotNull() 718 public ResultCode doToolProcessing() 719 { 720 int displayCount = reportCount.getValue(); 721 if (displayCount <= 0) 722 { 723 displayCount = Integer.MAX_VALUE; 724 } 725 726 String encryptionPassphrase = null; 727 if (encryptionPassphraseFile.isPresent()) 728 { 729 try 730 { 731 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 732 encryptionPassphraseFile.getValue()); 733 } 734 catch (final LDAPException e) 735 { 736 Debug.debugException(e); 737 err(e.getMessage()); 738 return e.getResultCode(); 739 } 740 } 741 742 743 long logLines = 0L; 744 for (final String path : argumentParser.getTrailingArguments()) 745 { 746 final File f = new File(path); 747 out("Examining access log ", f.getAbsolutePath()); 748 AccessLogReader reader = null; 749 InputStream inputStream = null; 750 try 751 { 752 inputStream = new FileInputStream(f); 753 754 final ObjectPair<InputStream,String> p = 755 ToolUtils.getPossiblyPassphraseEncryptedInputStream(inputStream, 756 encryptionPassphrase, 757 (! encryptionPassphraseFile.isPresent()), 758 "Log file '" + path + "' is encrypted. Please enter the " + 759 "encryption passphrase:", 760 "ERROR: The provided passphrase was incorrect.", 761 getOut(), getErr()); 762 inputStream = p.getFirst(); 763 if ((p.getSecond() != null) && (encryptionPassphrase == null)) 764 { 765 encryptionPassphrase = p.getSecond(); 766 } 767 768 if (isCompressed.isPresent()) 769 { 770 inputStream = new GZIPInputStream(inputStream); 771 } 772 else 773 { 774 inputStream = 775 ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream); 776 } 777 778 reader = new AccessLogReader(new InputStreamReader(inputStream)); 779 } 780 catch (final Exception e) 781 { 782 Debug.debugException(e); 783 err("Unable to open access log file ", f.getAbsolutePath(), ": ", 784 StaticUtils.getExceptionMessage(e)); 785 return ResultCode.LOCAL_ERROR; 786 } 787 finally 788 { 789 if ((reader == null) && (inputStream != null)) 790 { 791 try 792 { 793 inputStream.close(); 794 } 795 catch (final Exception e) 796 { 797 Debug.debugException(e); 798 } 799 } 800 } 801 802 long startTime = 0L; 803 long stopTime = 0L; 804 805 while (true) 806 { 807 final AccessLogMessage msg; 808 try 809 { 810 msg = reader.read(); 811 } 812 catch (final IOException ioe) 813 { 814 Debug.debugException(ioe); 815 err("Error reading from access log file ", f.getAbsolutePath(), 816 ": ", StaticUtils.getExceptionMessage(ioe)); 817 818 if ((ioe.getCause() != null) && 819 (ioe.getCause() instanceof BadPaddingException)) 820 { 821 err("This error is likely because the log is encrypted and the " + 822 "server still has the log file open. It is recommended " + 823 "that you only try to examine encrypted logs after they " + 824 "have been rotated. You can use the rotate-log tool to " + 825 "force a rotation at any time. Attempting to proceed with " + 826 "just the data that was successfully read."); 827 break; 828 } 829 else 830 { 831 return ResultCode.LOCAL_ERROR; 832 } 833 } 834 catch (final LogException le) 835 { 836 Debug.debugException(le); 837 err("Encountered an error while attempting to parse a line in" + 838 "access log file ", f.getAbsolutePath(), ": ", 839 StaticUtils.getExceptionMessage(le)); 840 continue; 841 } 842 843 if (msg == null) 844 { 845 break; 846 } 847 848 logLines++; 849 stopTime = msg.getTimestamp().getTime(); 850 if (startTime == 0L) 851 { 852 startTime = stopTime; 853 } 854 855 switch (msg.getMessageType()) 856 { 857 case CONNECT: 858 processConnect((ConnectAccessLogMessage) msg); 859 break; 860 case SECURITY_NEGOTIATION: 861 processSecurityNegotiation( 862 (SecurityNegotiationAccessLogMessage) msg); 863 break; 864 case DISCONNECT: 865 processDisconnect((DisconnectAccessLogMessage) msg); 866 break; 867 case REQUEST: 868 switch (((OperationAccessLogMessage) msg).getOperationType()) 869 { 870 case ABANDON: 871 processAbandonRequest((AbandonRequestAccessLogMessage) msg); 872 break; 873 case EXTENDED: 874 processExtendedRequest((ExtendedRequestAccessLogMessage) msg); 875 break; 876 case SEARCH: 877 processSearchRequest((SearchRequestAccessLogMessage) msg); 878 break; 879 case UNBIND: 880 processUnbindRequest((UnbindRequestAccessLogMessage) msg); 881 break; 882 } 883 break; 884 case RESULT: 885 switch (((OperationAccessLogMessage) msg).getOperationType()) 886 { 887 case ADD: 888 processAddResult((AddResultAccessLogMessage) msg); 889 break; 890 case BIND: 891 processBindResult((BindResultAccessLogMessage) msg); 892 break; 893 case COMPARE: 894 processCompareResult((CompareResultAccessLogMessage) msg); 895 break; 896 case DELETE: 897 processDeleteResult((DeleteResultAccessLogMessage) msg); 898 break; 899 case EXTENDED: 900 processExtendedResult((ExtendedResultAccessLogMessage) msg); 901 break; 902 case MODIFY: 903 processModifyResult((ModifyResultAccessLogMessage) msg); 904 break; 905 case MODDN: 906 processModifyDNResult((ModifyDNResultAccessLogMessage) msg); 907 break; 908 case SEARCH: 909 processSearchResult((SearchResultAccessLogMessage) msg); 910 break; 911 } 912 break; 913 914 case ASSURANCE_COMPLETE: 915 case CLIENT_CERTIFICATE: 916 case ENTRY_REBALANCING_REQUEST: 917 case ENTRY_REBALANCING_RESULT: 918 case FORWARD: 919 case FORWARD_FAILED: 920 case ENTRY: 921 case REFERENCE: 922 default: 923 // Nothing needs to be done for these message types. 924 } 925 } 926 927 try 928 { 929 reader.close(); 930 } 931 catch (final Exception e) 932 { 933 Debug.debugException(e); 934 } 935 logDurationMillis += (stopTime - startTime); 936 } 937 938 939 final int numFiles = argumentParser.getTrailingArguments().size(); 940 out(); 941 out("Examined ", logLines, " lines in ", numFiles, 942 ((numFiles == 1) ? " file" : " files"), 943 " covering a total duration of ", 944 StaticUtils.millisToHumanReadableDuration(logDurationMillis)); 945 if (logLines == 0) 946 { 947 return ResultCode.SUCCESS; 948 } 949 950 out(); 951 952 final double logDurationSeconds = logDurationMillis / 1_000.0; 953 final double connectsPerSecond = numConnects / logDurationSeconds; 954 final double disconnectsPerSecond = numDisconnects / logDurationSeconds; 955 956 out("Total connections established: ", numConnects, " (", 957 decimalFormat.format(connectsPerSecond), "/second)"); 958 out("Total disconnects: ", numDisconnects, " (", 959 decimalFormat.format(disconnectsPerSecond), "/second)"); 960 961 printCounts(clientAddresses, "Most common client addresses:", "address", 962 "addresses"); 963 964 printCounts(clientConnectionPolicies, 965 "Most common client connection policies:", "policy", "policies"); 966 967 printCounts(tlsProtocols, "Most common TLS protocol versions:", "version", 968 "versions"); 969 970 printCounts(tlsCipherSuites, "Most common TLS cipher suites:", 971 "cipher suite", "cipher suites"); 972 973 printCounts(disconnectReasons, "Most common disconnect reasons:", "reason", 974 "reasons"); 975 976 final long totalOps = numAbandons + numAdds + numBinds + numCompares + 977 numDeletes + numExtended + numModifies + numModifyDNs + numSearches + 978 numUnbinds; 979 if (totalOps > 0) 980 { 981 final double percentAbandon = 100.0 * numAbandons / totalOps; 982 final double percentAdd = 100.0 * numAdds / totalOps; 983 final double percentBind = 100.0 * numBinds / totalOps; 984 final double percentCompare = 100.0 * numCompares / totalOps; 985 final double percentDelete = 100.0 * numDeletes / totalOps; 986 final double percentExtended = 100.0 * numExtended / totalOps; 987 final double percentModify = 100.0 * numModifies / totalOps; 988 final double percentModifyDN = 100.0 * numModifyDNs / totalOps; 989 final double percentSearch = 100.0 * numSearches / totalOps; 990 final double percentUnbind = 100.0 * numUnbinds / totalOps; 991 992 final double abandonsPerSecond = numAbandons / logDurationSeconds; 993 final double addsPerSecond = numAdds / logDurationSeconds; 994 final double bindsPerSecond = numBinds / logDurationSeconds; 995 final double comparesPerSecond = numCompares / logDurationSeconds; 996 final double deletesPerSecond = numDeletes / logDurationSeconds; 997 final double extendedPerSecond = numExtended / logDurationSeconds; 998 final double modifiesPerSecond = numModifies / logDurationSeconds; 999 final double modifyDNsPerSecond = numModifyDNs / logDurationSeconds; 1000 final double searchesPerSecond = numSearches / logDurationSeconds; 1001 final double unbindsPerSecond = numUnbinds / logDurationSeconds; 1002 1003 out(); 1004 out("Total operations examined: ", totalOps); 1005 out("Abandon operations examined: ", numAbandons, " (", 1006 decimalFormat.format(percentAbandon), "%, ", 1007 decimalFormat.format(abandonsPerSecond), "/second)"); 1008 out("Add operations examined: ", numAdds, " (", 1009 decimalFormat.format(percentAdd), "%, ", 1010 decimalFormat.format(addsPerSecond), "/second)"); 1011 out("Bind operations examined: ", numBinds, " (", 1012 decimalFormat.format(percentBind), "%, ", 1013 decimalFormat.format(bindsPerSecond), "/second)"); 1014 out("Compare operations examined: ", numCompares, " (", 1015 decimalFormat.format(percentCompare), "%, ", 1016 decimalFormat.format(comparesPerSecond), "/second)"); 1017 out("Delete operations examined: ", numDeletes, " (", 1018 decimalFormat.format(percentDelete), "%, ", 1019 decimalFormat.format(deletesPerSecond), "/second)"); 1020 out("Extended operations examined: ", numExtended, " (", 1021 decimalFormat.format(percentExtended), "%, ", 1022 decimalFormat.format(extendedPerSecond), "/second)"); 1023 out("Modify operations examined: ", numModifies, " (", 1024 decimalFormat.format(percentModify), "%, ", 1025 decimalFormat.format(modifiesPerSecond), "/second)"); 1026 out("Modify DN operations examined: ", numModifyDNs, " (", 1027 decimalFormat.format(percentModifyDN), "%, ", 1028 decimalFormat.format(modifyDNsPerSecond), "/second)"); 1029 out("Search operations examined: ", numSearches, " (", 1030 decimalFormat.format(percentSearch), "%, ", 1031 decimalFormat.format(searchesPerSecond), "/second)"); 1032 out("Unbind operations examined: ", numUnbinds, " (", 1033 decimalFormat.format(percentUnbind), "%, ", 1034 decimalFormat.format(unbindsPerSecond), "/second)"); 1035 1036 final double totalProcessingDuration = addProcessingDuration + 1037 bindProcessingDuration + compareProcessingDuration + 1038 deleteProcessingDuration + extendedProcessingDuration + 1039 modifyProcessingDuration + modifyDNProcessingDuration + 1040 searchProcessingDuration; 1041 1042 out(); 1043 out("Average operation processing duration: ", 1044 decimalFormat.format(totalProcessingDuration / totalOps), "ms"); 1045 1046 if (numAdds > 0) 1047 { 1048 out("Average add operation processing duration: ", 1049 decimalFormat.format(addProcessingDuration / numAdds), "ms"); 1050 } 1051 1052 if (numBinds > 0) 1053 { 1054 out("Average bind operation processing duration: ", 1055 decimalFormat.format(bindProcessingDuration / numBinds), "ms"); 1056 } 1057 1058 if (numCompares > 0) 1059 { 1060 out("Average compare operation processing duration: ", 1061 decimalFormat.format(compareProcessingDuration / numCompares), 1062 "ms"); 1063 } 1064 1065 if (numDeletes > 0) 1066 { 1067 out("Average delete operation processing duration: ", 1068 decimalFormat.format(deleteProcessingDuration / numDeletes), "ms"); 1069 } 1070 1071 if (numExtended > 0) 1072 { 1073 out("Average extended operation processing duration: ", 1074 decimalFormat.format(extendedProcessingDuration / numExtended), 1075 "ms"); 1076 } 1077 1078 if (numModifies > 0) 1079 { 1080 out("Average modify operation processing duration: ", 1081 decimalFormat.format(modifyProcessingDuration / numModifies), "ms"); 1082 } 1083 1084 if (numModifyDNs > 0) 1085 { 1086 out("Average modify DN operation processing duration: ", 1087 decimalFormat.format(modifyDNProcessingDuration / numModifyDNs), 1088 "ms"); 1089 } 1090 1091 if (numSearches > 0) 1092 { 1093 out("Average search operation processing duration: ", 1094 decimalFormat.format(searchProcessingDuration / numSearches), "ms"); 1095 } 1096 1097 printProcessingTimeHistogram("add", numAdds, addProcessingTimes); 1098 printProcessingTimeHistogram("bind", numBinds, bindProcessingTimes); 1099 printProcessingTimeHistogram("compare", numCompares, 1100 compareProcessingTimes); 1101 printProcessingTimeHistogram("delete", numDeletes, deleteProcessingTimes); 1102 printProcessingTimeHistogram("extended", numExtended, 1103 extendedProcessingTimes); 1104 printProcessingTimeHistogram("modify", numModifies, 1105 modifyProcessingTimes); 1106 printProcessingTimeHistogram("modify DN", numModifyDNs, 1107 modifyDNProcessingTimes); 1108 printProcessingTimeHistogram("search", numSearches, 1109 searchProcessingTimes); 1110 1111 printResultCodeCounts(addResultCodes, "add"); 1112 printResultCodeCounts(bindResultCodes, "bind"); 1113 printResultCodeCounts(compareResultCodes, "compare"); 1114 printResultCodeCounts(deleteResultCodes, "delete"); 1115 printResultCodeCounts(extendedResultCodes, "extended"); 1116 printResultCodeCounts(modifyResultCodes, "modify"); 1117 printResultCodeCounts(modifyDNResultCodes, "modify DN"); 1118 printResultCodeCounts(searchResultCodes, "search"); 1119 1120 printCounts(successfulBindDNs, 1121 "Most common bind DNs used in successful authentication attempts:", 1122 "DN", "DNs"); 1123 printCounts(failedBindDNs, 1124 "Most common bind DNs used in failed authentication attempts:", 1125 "DN", "DNs"); 1126 printCounts(authenticationTypes, "Most common authentication types:", 1127 "authentication type", "authentication types"); 1128 1129 long numResultsWithAuthzID = 0L; 1130 for (final AtomicLong l : authzDNs.values()) 1131 { 1132 numResultsWithAuthzID += l.get(); 1133 } 1134 1135 out(); 1136 final double percentWithAuthzID = 1137 100.0 * numResultsWithAuthzID / totalOps; 1138 out("Number of operations with an alternate authorization identity: ", 1139 numResultsWithAuthzID, " (", 1140 decimalFormat.format(percentWithAuthzID), "%)"); 1141 1142 printCounts(authzDNs, "Most common alternate authorization identity DNs:", 1143 "DN", "DNs"); 1144 1145 if (! extendedOperations.isEmpty()) 1146 { 1147 final List<ObjectPair<String,Long>> extOpCounts = new ArrayList<>(); 1148 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1149 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1150 getMostCommonElements(extendedOperations, extOpCounts, displayCount, 1151 skippedWithSameCount, skippedWithLowerCount); 1152 1153 out(); 1154 out("Most common extended operation types:"); 1155 1156 long count = -1L; 1157 for (final ObjectPair<String,Long> p : extOpCounts) 1158 { 1159 count = p.getSecond(); 1160 final double percent = 100.0 * count / numExtended; 1161 1162 final String oid = p.getFirst(); 1163 final String name = extendedOperationOIDsToNames.get(oid); 1164 if (name == null) 1165 { 1166 out(p.getFirst(), ": ", p.getSecond(), " (", 1167 decimalFormat.format(percent), "%)"); 1168 } 1169 else 1170 { 1171 out(p.getFirst(), " (", name, "): ", p.getSecond(), " (", 1172 decimalFormat.format(percent), "%)"); 1173 } 1174 } 1175 1176 if (skippedWithSameCount.get() > 0L) 1177 { 1178 out("{ Skipped " + skippedWithSameCount.get() + 1179 " additional extended " + 1180 getSingularOrPlural(skippedWithSameCount.get(), "operation", 1181 "operations") + 1182 " with a count of " + count + " }"); 1183 } 1184 1185 if (skippedWithLowerCount.get() > 0L) 1186 { 1187 out("{ Skipped " + skippedWithLowerCount.get() + 1188 " additional extended " + 1189 getSingularOrPlural(skippedWithLowerCount.get(), "operation", 1190 "operations") + 1191 " with a count that is less than " + count + " }"); 1192 } 1193 } 1194 1195 out(); 1196 out("Number of unindexed search attempts: ", numUnindexedAttempts); 1197 out("Number of successfully-completed unindexed searches: ", 1198 numUnindexedSuccessful); 1199 out("Number of failed unindexed searches: ", numUnindexedFailed); 1200 1201 printCounts(unindexedFilters, "Most common unindexed search filters:", 1202 "filter", "filters"); 1203 1204 if (! searchScopes.isEmpty()) 1205 { 1206 final List<ObjectPair<SearchScope,Long>> scopeCounts = 1207 new ArrayList<>(); 1208 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1209 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1210 getMostCommonElements(searchScopes, scopeCounts, displayCount, 1211 skippedWithSameCount, skippedWithLowerCount); 1212 1213 out(); 1214 out("Most common search scopes:"); 1215 1216 long count = -1L; 1217 for (final ObjectPair<SearchScope,Long> p : scopeCounts) 1218 { 1219 count = p.getSecond(); 1220 final double percent = 100.0 * count / numSearches; 1221 out(p.getFirst().getName().toLowerCase(), " (", 1222 p.getFirst().intValue(), "): ", p.getSecond(), " (", 1223 decimalFormat.format(percent), "%)"); 1224 } 1225 1226 if (skippedWithSameCount.get() > 0L) 1227 { 1228 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1229 getSingularOrPlural(skippedWithSameCount.get(), "scope", 1230 "scopes") + 1231 " with a count of " + count + " }"); 1232 } 1233 1234 if (skippedWithLowerCount.get() > 0L) 1235 { 1236 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1237 getSingularOrPlural(skippedWithLowerCount.get(), "scope", 1238 "scopes") + 1239 " with a count that is less than " + count + " }"); 1240 } 1241 } 1242 1243 if (! searchEntryCounts.isEmpty()) 1244 { 1245 final List<ObjectPair<Long,Long>> entryCounts = new ArrayList<>(); 1246 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1247 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1248 getMostCommonElements(searchEntryCounts, entryCounts, displayCount, 1249 skippedWithSameCount, skippedWithLowerCount); 1250 1251 out(); 1252 out("Most common search entry counts:"); 1253 1254 long count = -1L; 1255 for (final ObjectPair<Long,Long> p : entryCounts) 1256 { 1257 count = p.getSecond(); 1258 final double percent = 100.0 * count / numSearches; 1259 out(p.getFirst(), " matching ", 1260 getSingularOrPlural(p.getFirst(), "entry", "entries"), 1261 ": ", p.getSecond(), " (", decimalFormat.format(percent), "%)"); 1262 } 1263 1264 if (skippedWithSameCount.get() > 0L) 1265 { 1266 out("{ Skipped " + skippedWithSameCount.get() + " additional entry " + 1267 getSingularOrPlural(skippedWithSameCount.get(), "count", 1268 "counts") + 1269 " with a count of " + count + " }"); 1270 } 1271 1272 if (skippedWithLowerCount.get() > 0L) 1273 { 1274 out("{ Skipped " + skippedWithLowerCount.get() + 1275 " additional entry " + 1276 getSingularOrPlural(skippedWithLowerCount.get(), "count", 1277 "counts") + 1278 " with a count that is less than " + count + " }"); 1279 } 1280 } 1281 1282 printCounts(searchBaseDNs, 1283 "Most common base DNs for searches with a non-base scope:", 1284 "base DN", "base DNs"); 1285 1286 printCounts(filterTypes, 1287 "Most common filters for searches with a non-base scope:", 1288 "filter", "filters"); 1289 1290 if (numSearches > 0L) 1291 { 1292 long numSearchesMatchingNoEntries = 0L; 1293 for (final AtomicLong l : noEntryFilters.values()) 1294 { 1295 numSearchesMatchingNoEntries += l.get(); 1296 } 1297 1298 out(); 1299 final double noEntryPercent = 1300 100.0 * numSearchesMatchingNoEntries / numSearches; 1301 out("Number of searches matching no entries: ", 1302 numSearchesMatchingNoEntries, " (", 1303 decimalFormat.format(noEntryPercent), "%)"); 1304 1305 printCounts(noEntryFilters, 1306 "Most common filters for searches matching no entries:", 1307 "filter", "filters"); 1308 1309 1310 long numSearchesMatchingOneEntry = 0L; 1311 for (final AtomicLong l : oneEntryFilters.values()) 1312 { 1313 numSearchesMatchingOneEntry += l.get(); 1314 } 1315 1316 out(); 1317 final double oneEntryPercent = 1318 100.0 * numSearchesMatchingOneEntry / numSearches; 1319 out("Number of searches matching one entry: ", 1320 numSearchesMatchingOneEntry, " (", 1321 decimalFormat.format(oneEntryPercent), "%)"); 1322 1323 printCounts(oneEntryFilters, 1324 "Most common filters for searches matching one entry:", 1325 "filter", "filters"); 1326 1327 1328 long numSearchesMatchingMultipleEntries = 0L; 1329 for (final AtomicLong l : multiEntryFilters.values()) 1330 { 1331 numSearchesMatchingMultipleEntries += l.get(); 1332 } 1333 1334 out(); 1335 final double multiEntryPercent = 1336 100.0 * numSearchesMatchingMultipleEntries / numSearches; 1337 out("Number of searches matching multiple entries: ", 1338 numSearchesMatchingMultipleEntries, " (", 1339 decimalFormat.format(multiEntryPercent), "%)"); 1340 1341 printCounts(multiEntryFilters, 1342 "Most common filters for searches matching multiple entries:", 1343 "filter", "filters"); 1344 } 1345 } 1346 1347 if (! mostExpensiveFilters.isEmpty()) 1348 { 1349 final List<ObjectPair<String,Long>> filterDurations = new ArrayList<>(); 1350 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1351 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1352 getMostCommonElements(mostExpensiveFilters, filterDurations, 1353 displayCount, skippedWithSameCount, skippedWithLowerCount); 1354 1355 out(); 1356 out("Filters for searches with the longest processing times:"); 1357 1358 String durationStr = ""; 1359 for (final ObjectPair<String,Long> p : filterDurations) 1360 { 1361 final long durationMicros = p.getSecond(); 1362 final double durationMillis = durationMicros / 1_000.0; 1363 durationStr = decimalFormat.format(durationMillis) + " ms"; 1364 out(p.getFirst(), ": ", durationStr); 1365 } 1366 1367 if (skippedWithSameCount.get() > 0L) 1368 { 1369 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1370 getSingularOrPlural(skippedWithSameCount.get(), "filter", 1371 "filters") + 1372 " with a duration of " + durationStr + " }"); 1373 } 1374 1375 if (skippedWithLowerCount.get() > 0L) 1376 { 1377 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1378 getSingularOrPlural(skippedWithLowerCount.get(), "filter", 1379 "filters") + 1380 " with a duration that is less than " + durationStr + " }"); 1381 } 1382 } 1383 1384 final long totalUncached = numUncachedAdds + numUncachedBinds + 1385 numUncachedCompares + numUncachedDeletes + numUncachedExtended + 1386 numUncachedModifies + numUncachedModifyDNs + numUncachedSearches; 1387 if (totalUncached > 0L) 1388 { 1389 out(); 1390 out("Operations accessing uncached data:"); 1391 printUncached("Add", numUncachedAdds, numAdds); 1392 printUncached("Bind", numUncachedBinds, numBinds); 1393 printUncached("Compare", numUncachedCompares, numCompares); 1394 printUncached("Delete", numUncachedDeletes, numDeletes); 1395 printUncached("Extended", numUncachedExtended, numExtended); 1396 printUncached("Modify", numUncachedModifies, numModifies); 1397 printUncached("Modify DN", numUncachedModifyDNs, numModifyDNs); 1398 printUncached("Search", numUncachedSearches, numSearches); 1399 } 1400 1401 1402 return ResultCode.SUCCESS; 1403 } 1404 1405 1406 1407 /** 1408 * Retrieves a set of information that may be used to generate example usage 1409 * information. Each element in the returned map should consist of a map 1410 * between an example set of arguments and a string that describes the 1411 * behavior of the tool when invoked with that set of arguments. 1412 * 1413 * @return A set of information that may be used to generate example usage 1414 * information. It may be {@code null} or empty if no example usage 1415 * information is available. 1416 */ 1417 @Override() 1418 @NotNull() 1419 public LinkedHashMap<String[],String> getExampleUsages() 1420 { 1421 final LinkedHashMap<String[],String> examples = 1422 new LinkedHashMap<>(StaticUtils.computeMapCapacity(1)); 1423 1424 final String[] args = 1425 { 1426 "/ds/logs/access" 1427 }; 1428 final String description = 1429 "Analyze the contents of the /ds/logs/access access log file."; 1430 examples.put(args, description); 1431 1432 return examples; 1433 } 1434 1435 1436 1437 /** 1438 * Populates the provided processing time map with an initial set of values. 1439 * 1440 * @param m The processing time map to be populated. 1441 */ 1442 private static void populateProcessingTimeMap( 1443 @NotNull final HashMap<Long,AtomicLong> m) 1444 { 1445 m.put(1L, new AtomicLong(0L)); 1446 m.put(2L, new AtomicLong(0L)); 1447 m.put(3L, new AtomicLong(0L)); 1448 m.put(5L, new AtomicLong(0L)); 1449 m.put(10L, new AtomicLong(0L)); 1450 m.put(20L, new AtomicLong(0L)); 1451 m.put(30L, new AtomicLong(0L)); 1452 m.put(50L, new AtomicLong(0L)); 1453 m.put(100L, new AtomicLong(0L)); 1454 m.put(1_000L, new AtomicLong(0L)); 1455 m.put(2_000L, new AtomicLong(0L)); 1456 m.put(3_000L, new AtomicLong(0L)); 1457 m.put(5_000L, new AtomicLong(0L)); 1458 m.put(10_000L, new AtomicLong(0L)); 1459 m.put(20_000L, new AtomicLong(0L)); 1460 m.put(30_000L, new AtomicLong(0L)); 1461 m.put(60_000L, new AtomicLong(0L)); 1462 m.put(Long.MAX_VALUE, new AtomicLong(0L)); 1463 } 1464 1465 1466 1467 /** 1468 * Performs any necessary processing for a connect message. 1469 * 1470 * @param m The log message to be processed. 1471 */ 1472 private void processConnect(@NotNull final ConnectAccessLogMessage m) 1473 { 1474 numConnects++; 1475 1476 final String clientAddr = m.getSourceAddress(); 1477 if (clientAddr != null) 1478 { 1479 AtomicLong count = clientAddresses.get(clientAddr); 1480 if (count == null) 1481 { 1482 count = new AtomicLong(0L); 1483 clientAddresses.put(clientAddr, count); 1484 } 1485 count.incrementAndGet(); 1486 } 1487 1488 final String ccp = m.getClientConnectionPolicy(); 1489 if (ccp != null) 1490 { 1491 AtomicLong l = clientConnectionPolicies.get(ccp); 1492 if (l == null) 1493 { 1494 l = new AtomicLong(0L); 1495 clientConnectionPolicies.put(ccp, l); 1496 } 1497 l.incrementAndGet(); 1498 } 1499 } 1500 1501 1502 1503 /** 1504 * Performs any necessary processing for a security negotiation message. 1505 * 1506 * @param m The log message to be processed. 1507 */ 1508 private void processSecurityNegotiation( 1509 @NotNull final SecurityNegotiationAccessLogMessage m) 1510 { 1511 final String protocol = m.getProtocol(); 1512 if (protocol != null) 1513 { 1514 AtomicLong l = tlsProtocols.get(protocol); 1515 if (l == null) 1516 { 1517 l = new AtomicLong(0L); 1518 tlsProtocols.put(protocol, l); 1519 } 1520 l.incrementAndGet(); 1521 } 1522 1523 final String cipherSuite = m.getCipher(); 1524 if (cipherSuite != null) 1525 { 1526 AtomicLong l = tlsCipherSuites.get(cipherSuite); 1527 if (l == null) 1528 { 1529 l = new AtomicLong(0L); 1530 tlsCipherSuites.put(cipherSuite, l); 1531 } 1532 l.incrementAndGet(); 1533 } 1534 } 1535 1536 1537 1538 /** 1539 * Performs any necessary processing for a disconnect message. 1540 * 1541 * @param m The log message to be processed. 1542 */ 1543 private void processDisconnect(@NotNull final DisconnectAccessLogMessage m) 1544 { 1545 numDisconnects++; 1546 1547 final String reason = m.getDisconnectReason(); 1548 if (reason != null) 1549 { 1550 AtomicLong l = disconnectReasons.get(reason); 1551 if (l == null) 1552 { 1553 l = new AtomicLong(0L); 1554 disconnectReasons.put(reason, l); 1555 } 1556 l.incrementAndGet(); 1557 } 1558 } 1559 1560 1561 1562 /** 1563 * Performs any necessary processing for an abandon request message. 1564 * 1565 * @param m The log message to be processed. 1566 */ 1567 private void processAbandonRequest( 1568 @NotNull final AbandonRequestAccessLogMessage m) 1569 { 1570 numAbandons++; 1571 } 1572 1573 1574 1575 /** 1576 * Performs any necessary processing for an extended request message. 1577 * 1578 * @param m The log message to be processed. 1579 */ 1580 private void processExtendedRequest( 1581 @NotNull final ExtendedRequestAccessLogMessage m) 1582 { 1583 processedRequests.add(m.getConnectionID() + "-" + m.getOperationID()); 1584 processExtendedRequestInternal(m); 1585 } 1586 1587 1588 1589 /** 1590 * Performs the internal processing for an extended request message. 1591 * 1592 * @param m The log message to be processed. 1593 */ 1594 private void processExtendedRequestInternal( 1595 @NotNull final ExtendedRequestAccessLogMessage m) 1596 { 1597 final String oid = m.getRequestOID(); 1598 if (oid != null) 1599 { 1600 AtomicLong l = extendedOperations.get(oid); 1601 if (l == null) 1602 { 1603 l = new AtomicLong(0L); 1604 extendedOperations.put(oid, l); 1605 } 1606 l.incrementAndGet(); 1607 1608 final String requestType = m.getRequestType(); 1609 if ((requestType != null) && 1610 (! extendedOperationOIDsToNames.containsKey(oid))) 1611 { 1612 extendedOperationOIDsToNames.put(oid, requestType); 1613 } 1614 } 1615 } 1616 1617 1618 1619 /** 1620 * Performs any necessary processing for a search request message. 1621 * 1622 * @param m The log message to be processed. 1623 */ 1624 private void processSearchRequest( 1625 @NotNull final SearchRequestAccessLogMessage m) 1626 { 1627 processedRequests.add(m.getConnectionID() + "-" + m.getOperationID()); 1628 processSearchRequestInternal(m); 1629 } 1630 1631 1632 1633 /** 1634 * Performs any necessary processing for a search request message. 1635 * 1636 * @param m The log message to be processed. 1637 */ 1638 private void processSearchRequestInternal( 1639 @NotNull final SearchRequestAccessLogMessage m) 1640 { 1641 final SearchScope scope = m.getScope(); 1642 if (scope != null) 1643 { 1644 AtomicLong scopeCount = searchScopes.get(scope); 1645 if (scopeCount == null) 1646 { 1647 scopeCount = new AtomicLong(0L); 1648 searchScopes.put(scope, scopeCount); 1649 } 1650 scopeCount.incrementAndGet(); 1651 1652 if (! scope.equals(SearchScope.BASE)) 1653 { 1654 final String filterString = getFilterString(m.getParsedFilter()); 1655 if (filterString != null) 1656 { 1657 AtomicLong filterCount = filterTypes.get(filterString); 1658 if (filterCount == null) 1659 { 1660 filterCount = new AtomicLong(0L); 1661 filterTypes.put(filterString, filterCount); 1662 } 1663 filterCount.incrementAndGet(); 1664 1665 1666 final String baseDN = getDNString(m.getBaseDN()); 1667 if (baseDN != null) 1668 { 1669 AtomicLong baseDNCount = searchBaseDNs.get(baseDN); 1670 if (baseDNCount == null) 1671 { 1672 baseDNCount = new AtomicLong(0L); 1673 searchBaseDNs.put(baseDN, baseDNCount); 1674 } 1675 baseDNCount.incrementAndGet(); 1676 } 1677 } 1678 } 1679 } 1680 } 1681 1682 1683 1684 /** 1685 * Performs any necessary processing for an unbind request message. 1686 * 1687 * @param m The log message to be processed. 1688 */ 1689 private void processUnbindRequest( 1690 @NotNull final UnbindRequestAccessLogMessage m) 1691 { 1692 numUnbinds++; 1693 } 1694 1695 1696 1697 /** 1698 * Performs any necessary processing for an add result message. 1699 * 1700 * @param m The log message to be processed. 1701 */ 1702 private void processAddResult(@NotNull final AddResultAccessLogMessage m) 1703 { 1704 numAdds++; 1705 1706 updateResultCodeCount(m.getResultCode(), addResultCodes); 1707 addProcessingDuration += 1708 doubleValue(m.getProcessingTimeMillis(), addProcessingTimes); 1709 1710 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 1711 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 1712 { 1713 numUncachedAdds++; 1714 } 1715 1716 updateAuthzCount(m.getAlternateAuthorizationDN()); 1717 } 1718 1719 1720 1721 /** 1722 * Performs any necessary processing for a bind result message. 1723 * 1724 * @param m The log message to be processed. 1725 */ 1726 private void processBindResult(@NotNull final BindResultAccessLogMessage m) 1727 { 1728 numBinds++; 1729 1730 if (m.getAuthenticationType() != null) 1731 { 1732 final String authType; 1733 switch (m.getAuthenticationType()) 1734 { 1735 case SIMPLE: 1736 authType = "Simple"; 1737 break; 1738 1739 case SASL: 1740 final String saslMechanism = m.getSASLMechanismName(); 1741 if (saslMechanism == null) 1742 { 1743 authType = "SASL {unknown mechanism}"; 1744 } 1745 else 1746 { 1747 authType = "SASL " + saslMechanism; 1748 } 1749 break; 1750 1751 case INTERNAL: 1752 authType = "Internal"; 1753 break; 1754 1755 default: 1756 authType = m.getAuthenticationType().name(); 1757 break; 1758 } 1759 1760 AtomicLong l = authenticationTypes.get(authType); 1761 if (l == null) 1762 { 1763 l = new AtomicLong(0L); 1764 authenticationTypes.put(authType, l); 1765 } 1766 l.incrementAndGet(); 1767 } 1768 1769 updateResultCodeCount(m.getResultCode(), bindResultCodes); 1770 bindProcessingDuration += 1771 doubleValue(m.getProcessingTimeMillis(), bindProcessingTimes); 1772 1773 String authenticationDN = getDNString(m.getAuthenticationDN()); 1774 if (m.getResultCode() == ResultCode.SUCCESS) 1775 { 1776 if (authenticationDN != null) 1777 { 1778 AtomicLong l = successfulBindDNs.get(authenticationDN); 1779 if (l == null) 1780 { 1781 l = new AtomicLong(0L); 1782 successfulBindDNs.put(authenticationDN, l); 1783 } 1784 l.incrementAndGet(); 1785 } 1786 1787 final String ccp = m.getClientConnectionPolicy(); 1788 if (ccp != null) 1789 { 1790 AtomicLong l = clientConnectionPolicies.get(ccp); 1791 if (l == null) 1792 { 1793 l = new AtomicLong(0L); 1794 clientConnectionPolicies.put(ccp, l); 1795 } 1796 l.incrementAndGet(); 1797 } 1798 } 1799 else if ((m.getResultCode() != ResultCode.SASL_BIND_IN_PROGRESS) && 1800 (m.getResultCode() != ResultCode.REFERRAL)) 1801 { 1802 if (authenticationDN == null) 1803 { 1804 authenticationDN = getDNString(m.getDN()); 1805 } 1806 1807 if (authenticationDN != null) 1808 { 1809 AtomicLong l = failedBindDNs.get(authenticationDN); 1810 if (l == null) 1811 { 1812 l = new AtomicLong(0L); 1813 failedBindDNs.put(authenticationDN, l); 1814 } 1815 l.incrementAndGet(); 1816 } 1817 } 1818 1819 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 1820 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 1821 { 1822 numUncachedBinds++; 1823 } 1824 1825 updateAuthzCount(m.getAuthorizationDN()); 1826 } 1827 1828 1829 1830 /** 1831 * Performs any necessary processing for a compare result message. 1832 * 1833 * @param m The log message to be processed. 1834 */ 1835 private void processCompareResult( 1836 @NotNull final CompareResultAccessLogMessage m) 1837 { 1838 numCompares++; 1839 1840 updateResultCodeCount(m.getResultCode(), compareResultCodes); 1841 compareProcessingDuration += 1842 doubleValue(m.getProcessingTimeMillis(), compareProcessingTimes); 1843 1844 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 1845 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 1846 { 1847 numUncachedCompares++; 1848 } 1849 1850 updateAuthzCount(m.getAlternateAuthorizationDN()); 1851 } 1852 1853 1854 1855 /** 1856 * Performs any necessary processing for a delete result message. 1857 * 1858 * @param m The log message to be processed. 1859 */ 1860 private void processDeleteResult( 1861 @NotNull final DeleteResultAccessLogMessage m) 1862 { 1863 numDeletes++; 1864 1865 updateResultCodeCount(m.getResultCode(), deleteResultCodes); 1866 deleteProcessingDuration += 1867 doubleValue(m.getProcessingTimeMillis(), deleteProcessingTimes); 1868 1869 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 1870 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 1871 { 1872 numUncachedDeletes++; 1873 } 1874 1875 updateAuthzCount(m.getAlternateAuthorizationDN()); 1876 } 1877 1878 1879 1880 /** 1881 * Performs any necessary processing for an extended result message. 1882 * 1883 * @param m The log message to be processed. 1884 */ 1885 private void processExtendedResult( 1886 @NotNull final ExtendedResultAccessLogMessage m) 1887 { 1888 numExtended++; 1889 1890 final String id = m.getConnectionID() + "-" + m.getOperationID(); 1891 if (!processedRequests.remove(id)) 1892 { 1893 processExtendedRequestInternal(m); 1894 } 1895 1896 updateResultCodeCount(m.getResultCode(), extendedResultCodes); 1897 extendedProcessingDuration += 1898 doubleValue(m.getProcessingTimeMillis(), extendedProcessingTimes); 1899 1900 final String ccp = m.getClientConnectionPolicy(); 1901 if (ccp != null) 1902 { 1903 AtomicLong l = clientConnectionPolicies.get(ccp); 1904 if (l == null) 1905 { 1906 l = new AtomicLong(0L); 1907 clientConnectionPolicies.put(ccp, l); 1908 } 1909 l.incrementAndGet(); 1910 } 1911 1912 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 1913 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 1914 { 1915 numUncachedExtended++; 1916 } 1917 } 1918 1919 1920 1921 /** 1922 * Performs any necessary processing for a modify result message. 1923 * 1924 * @param m The log message to be processed. 1925 */ 1926 private void processModifyResult( 1927 @NotNull final ModifyResultAccessLogMessage m) 1928 { 1929 numModifies++; 1930 1931 updateResultCodeCount(m.getResultCode(), modifyResultCodes); 1932 modifyProcessingDuration += 1933 doubleValue(m.getProcessingTimeMillis(), modifyProcessingTimes); 1934 1935 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 1936 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 1937 { 1938 numUncachedModifies++; 1939 } 1940 1941 updateAuthzCount(m.getAlternateAuthorizationDN()); 1942 } 1943 1944 1945 1946 /** 1947 * Performs any necessary processing for a modify DN result message. 1948 * 1949 * @param m The log message to be processed. 1950 */ 1951 private void processModifyDNResult( 1952 @NotNull final ModifyDNResultAccessLogMessage m) 1953 { 1954 numModifyDNs++; 1955 1956 updateResultCodeCount(m.getResultCode(), modifyDNResultCodes); 1957 modifyDNProcessingDuration += 1958 doubleValue(m.getProcessingTimeMillis(), modifyDNProcessingTimes); 1959 1960 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 1961 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 1962 { 1963 numUncachedModifyDNs++; 1964 } 1965 1966 updateAuthzCount(m.getAlternateAuthorizationDN()); 1967 } 1968 1969 1970 1971 /** 1972 * Performs any necessary processing for a search result message. 1973 * 1974 * @param m The log message to be processed. 1975 */ 1976 private void processSearchResult( 1977 @NotNull final SearchResultAccessLogMessage m) 1978 { 1979 numSearches++; 1980 1981 final String id = m.getConnectionID() + "-" + m.getOperationID(); 1982 if (! processedRequests.remove(id)) 1983 { 1984 processSearchRequestInternal(m); 1985 } 1986 1987 final ResultCode resultCode = m.getResultCode(); 1988 updateResultCodeCount(resultCode, searchResultCodes); 1989 searchProcessingDuration += 1990 doubleValue(m.getProcessingTimeMillis(), searchProcessingTimes); 1991 1992 final String filterString = getFilterString(m.getParsedFilter()); 1993 1994 final Long entryCount = m.getEntriesReturned(); 1995 if (entryCount != null) 1996 { 1997 AtomicLong l = searchEntryCounts.get(entryCount); 1998 if (l == null) 1999 { 2000 l = new AtomicLong(0L); 2001 searchEntryCounts.put(entryCount, l); 2002 } 2003 l.incrementAndGet(); 2004 2005 final Map<String,AtomicLong> filterCountMap; 2006 switch (entryCount.intValue()) 2007 { 2008 case 0: 2009 filterCountMap = noEntryFilters; 2010 break; 2011 case 1: 2012 filterCountMap = oneEntryFilters; 2013 break; 2014 default: 2015 filterCountMap = multiEntryFilters; 2016 break; 2017 } 2018 2019 if (filterString != null) 2020 { 2021 AtomicLong filterCount = filterCountMap.get(filterString); 2022 if (filterCount == null) 2023 { 2024 filterCount = new AtomicLong(0L); 2025 filterCountMap.put(filterString, filterCount); 2026 } 2027 filterCount.incrementAndGet(); 2028 } 2029 } 2030 2031 final Boolean isUnindexed = m.isUnindexed(); 2032 if ((isUnindexed != null) && isUnindexed) 2033 { 2034 numUnindexedAttempts++; 2035 if (resultCode == ResultCode.SUCCESS) 2036 { 2037 numUnindexedSuccessful++; 2038 } 2039 else 2040 { 2041 numUnindexedFailed++; 2042 } 2043 2044 if (filterString != null) 2045 { 2046 AtomicLong l = unindexedFilters.get(filterString); 2047 if (l == null) 2048 { 2049 l = new AtomicLong(0L); 2050 unindexedFilters.put(filterString, l); 2051 } 2052 l.incrementAndGet(); 2053 } 2054 } 2055 2056 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2057 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2058 { 2059 numUncachedSearches++; 2060 } 2061 2062 updateAuthzCount(m.getAlternateAuthorizationDN()); 2063 2064 final Double processingTimeMillis = m.getProcessingTimeMillis(); 2065 if ((processingTimeMillis != null) && (filterString != null)) 2066 { 2067 final long processingTimeMicros = 2068 Math.round(processingTimeMillis * 1_000.0); 2069 2070 AtomicLong l = mostExpensiveFilters.get(filterString); 2071 if (l == null) 2072 { 2073 l = new AtomicLong(processingTimeMicros); 2074 mostExpensiveFilters.put(filterString, l); 2075 } 2076 else 2077 { 2078 final long previousProcessingTimeMicros = l.get(); 2079 if (processingTimeMicros > previousProcessingTimeMicros) 2080 { 2081 l.set(processingTimeMicros); 2082 } 2083 } 2084 } 2085 } 2086 2087 2088 2089 /** 2090 * Updates the count for the provided result code in the given map. 2091 * 2092 * @param rc The result code for which to update the count. 2093 * @param m The map used to hold counts by result code. 2094 */ 2095 private static void updateResultCodeCount(@Nullable final ResultCode rc, 2096 @NotNull final HashMap<ResultCode,AtomicLong> m) 2097 { 2098 if (rc == null) 2099 { 2100 return; 2101 } 2102 2103 AtomicLong l = m.get(rc); 2104 if (l == null) 2105 { 2106 l = new AtomicLong(0L); 2107 m.put(rc, l); 2108 } 2109 l.incrementAndGet(); 2110 } 2111 2112 2113 2114 /** 2115 * Retrieves the double value for the provided {@code Double} object. 2116 * 2117 * @param d The {@code Double} object for which to retrieve the value. 2118 * @param m The processing time histogram map to be updated. 2119 * 2120 * @return The double value of the provided {@code Double} object if it was 2121 * non-{@code null}, or 0.0 if it was {@code null}. 2122 */ 2123 private static double doubleValue(@Nullable final Double d, 2124 @NotNull final HashMap<Long,AtomicLong> m) 2125 { 2126 if (d == null) 2127 { 2128 return 0.0; 2129 } 2130 else 2131 { 2132 for (final Map.Entry<Long,AtomicLong> e : m.entrySet()) 2133 { 2134 if (d <= e.getKey()) 2135 { 2136 e.getValue().incrementAndGet(); 2137 break; 2138 } 2139 } 2140 2141 return d; 2142 } 2143 } 2144 2145 2146 2147 /** 2148 * Updates the provided list with the most frequently-occurring elements in 2149 * the provided map, paired with the number of times each value occurred. 2150 * 2151 * @param <K> The type of object used as the key for the 2152 * provided map. 2153 * @param countMap The map to be examined. It is expected that 2154 * the values of the map will be the count of 2155 * occurrences for the keys. 2156 * @param mostCommonElementList The list to which the values will be 2157 * updated. It must not be {@code null}, must 2158 * be empty, and must be updatable. 2159 * @param maxListSize The maximum number of items to add to the 2160 * provided list. It must be greater than 2161 * zero. 2162 * @param skippedWithSameCount A counter that will be incremented for each 2163 * map entry that is skipped with the same 2164 * count as a value that was not skipped. It 2165 * must not be {@code null} and must initially 2166 * be zero. 2167 * @param skippedWithLowerCount A counter that will be incremented for each 2168 * map entry that is skipped with a lower count 2169 * as the last value that was not skipped. It 2170 * must not be {@code null} and must initially 2171 * be zero. 2172 * 2173 * @return A list of the most frequently-occurring elements in the provided 2174 * map. 2175 */ 2176 @NotNull() 2177 private static <K> List<ObjectPair<K,Long>> getMostCommonElements( 2178 @NotNull final Map<K,AtomicLong> countMap, 2179 @NotNull final List<ObjectPair<K,Long>> mostCommonElementList, 2180 final int maxListSize, 2181 @NotNull final AtomicLong skippedWithSameCount, 2182 @NotNull final AtomicLong skippedWithLowerCount) 2183 { 2184 final TreeMap<Long,List<K>> reverseMap = 2185 new TreeMap<>(new ReverseComparator<Long>()); 2186 for (final Map.Entry<K,AtomicLong> e : countMap.entrySet()) 2187 { 2188 final Long count = e.getValue().get(); 2189 List<K> list = reverseMap.get(count); 2190 if (list == null) 2191 { 2192 list = new ArrayList<>(); 2193 reverseMap.put(count, list); 2194 } 2195 list.add(e.getKey()); 2196 } 2197 2198 for (final Map.Entry<Long,List<K>> e : reverseMap.entrySet()) 2199 { 2200 final Long l = e.getKey(); 2201 int numNotSkipped = 0; 2202 for (final K k : e.getValue()) 2203 { 2204 if (mostCommonElementList.size() >= maxListSize) 2205 { 2206 if (numNotSkipped > 0) 2207 { 2208 skippedWithSameCount.incrementAndGet(); 2209 } 2210 else 2211 { 2212 skippedWithLowerCount.incrementAndGet(); 2213 } 2214 } 2215 else 2216 { 2217 numNotSkipped++; 2218 mostCommonElementList.add(new ObjectPair<>(k, l)); 2219 } 2220 } 2221 } 2222 2223 return mostCommonElementList; 2224 } 2225 2226 2227 2228 /** 2229 * Updates the count of alternate authorization identities for the provided 2230 * DN. 2231 * 2232 * @param authzDN The DN of the alternate authorization identity that was 2233 * used. It may be {@code null} if no alternate 2234 * authorization identity was used. 2235 */ 2236 private void updateAuthzCount(@Nullable final String authzDN) 2237 { 2238 if (authzDN == null) 2239 { 2240 return; 2241 } 2242 2243 final String dnString = getDNString(authzDN); 2244 2245 AtomicLong l = authzDNs.get(dnString); 2246 if (l == null) 2247 { 2248 l = new AtomicLong(0L); 2249 authzDNs.put(dnString, l); 2250 } 2251 } 2252 2253 2254 2255 /** 2256 * Retrieves a string representation of the provided DN. It may either be 2257 * anonymized, using question marks in place of specific attribute values, or 2258 * it may be the actual string representation of the given DN. 2259 * 2260 * @param dn The DN for which to retrieve the string representation. 2261 * 2262 * @return A string representation of the provided DN, or {@code null} if the 2263 * given DN was {@code null}. 2264 */ 2265 @Nullable() 2266 private String getDNString(@Nullable final String dn) 2267 { 2268 if (dn == null) 2269 { 2270 return null; 2271 } 2272 2273 final DN parsedDN; 2274 try 2275 { 2276 parsedDN = new DN(dn); 2277 } 2278 catch (final Exception e) 2279 { 2280 Debug.debugException(e); 2281 return dn.toLowerCase(); 2282 } 2283 2284 if (parsedDN.isNullDN()) 2285 { 2286 return "{Null DN}"; 2287 } 2288 2289 if (doNotAnonymize.isPresent()) 2290 { 2291 return parsedDN.toNormalizedString(); 2292 } 2293 2294 final StringBuilder buffer = new StringBuilder(); 2295 final RDN[] rdns = parsedDN.getRDNs(); 2296 for (int i=0; i < rdns.length; i++) 2297 { 2298 if (i > 0) 2299 { 2300 buffer.append(','); 2301 } 2302 2303 final RDN rdn = rdns[i]; 2304 final String[] attributeNames = rdn.getAttributeNames(); 2305 for (int j=0; j < attributeNames.length; j++) 2306 { 2307 if (j > 0) 2308 { 2309 buffer.append('+'); 2310 } 2311 buffer.append(attributeNames[j].toLowerCase()); 2312 buffer.append("=?"); 2313 } 2314 } 2315 2316 return buffer.toString(); 2317 } 2318 2319 2320 2321 /** 2322 * Retrieves a string representation of the provided filter. It may 2323 * potentially be de-anonymized to include specific values. 2324 * 2325 * @param filter The filter for which to obtain the string representation. 2326 * 2327 * @return A string representation of the provided filter (which may or may 2328 * not be anonymized), or {@code null} if the provided filter is 2329 * {@code null}. 2330 */ 2331 @Nullable() 2332 private String getFilterString(@Nullable final Filter filter) 2333 { 2334 if (filter == null) 2335 { 2336 return null; 2337 } 2338 2339 if (doNotAnonymize.isPresent()) 2340 { 2341 return filter.toString().toLowerCase(); 2342 } 2343 2344 return new GenericFilter(filter).toString().toLowerCase(); 2345 } 2346 2347 2348 2349 /** 2350 * Writes a breakdown of the processing times for a specified type of 2351 * operation. 2352 * 2353 * @param t The name of the operation type. 2354 * @param n The total number of operations of the specified type that were 2355 * processed by the server. 2356 * @param m The map of operation counts by processing time bucket. 2357 */ 2358 private void printProcessingTimeHistogram(@NotNull final String t, 2359 final long n, 2360 @NotNull final LinkedHashMap<Long,AtomicLong> m) 2361 { 2362 if (n <= 0) 2363 { 2364 return; 2365 } 2366 2367 out(); 2368 out("Count of ", t, " operations by processing time:"); 2369 2370 long lowerBound = 0; 2371 long accumulatedCount = 0; 2372 final Iterator<Map.Entry<Long,AtomicLong>> i = m.entrySet().iterator(); 2373 while (i.hasNext()) 2374 { 2375 final Map.Entry<Long,AtomicLong> e = i.next(); 2376 final long upperBound = e.getKey(); 2377 final long count = e.getValue().get(); 2378 final double categoryPercent = 100.0 * count / n; 2379 2380 accumulatedCount += count; 2381 final double accumulatedPercent = 100.0 * accumulatedCount / n; 2382 2383 if (i.hasNext()) 2384 { 2385 final String lowerBoundString; 2386 if (lowerBound == 0L) 2387 { 2388 lowerBoundString = "0 milliseconds"; 2389 } 2390 else 2391 { 2392 final long lowerBoundNanos = lowerBound * 1_000_000L; 2393 lowerBoundString = DurationArgument.nanosToDuration(lowerBoundNanos); 2394 } 2395 2396 final long upperBoundNanos = upperBound * 1_000_000L; 2397 final String upperBoundString = 2398 DurationArgument.nanosToDuration(upperBoundNanos); 2399 2400 2401 out("Between ", lowerBoundString, " and ", upperBoundString, ": ", 2402 count, " (", decimalFormat.format(categoryPercent), "%, ", 2403 decimalFormat.format(accumulatedPercent), "% accumulated)"); 2404 lowerBound = upperBound; 2405 } 2406 else 2407 { 2408 final long lowerBoundNanos = lowerBound * 1_000_000L; 2409 final String lowerBoundString = 2410 DurationArgument.nanosToDuration(lowerBoundNanos); 2411 2412 out("Greater than ", lowerBoundString, ": ", count, " (", 2413 decimalFormat.format(categoryPercent), "%, ", 2414 decimalFormat.format(accumulatedPercent), "% accumulated)"); 2415 } 2416 } 2417 } 2418 2419 2420 2421 /** 2422 * Optionally prints information about the number and percent of operations of 2423 * the specified type that involved access to uncached data. 2424 * 2425 * @param operationType The type of operation. 2426 * @param numUncached The number of operations of the specified type that 2427 * involved access to uncached data. 2428 * @param numTotal The total number of operations of the specified 2429 * type. 2430 */ 2431 private void printUncached(@NotNull final String operationType, 2432 final long numUncached, 2433 final long numTotal) 2434 { 2435 if (numUncached == 0) 2436 { 2437 return; 2438 } 2439 2440 out(operationType, ": ", numUncached, " (", 2441 decimalFormat.format(100.0 * numUncached / numTotal), "%)"); 2442 } 2443 2444 2445 2446 /** 2447 * Prints data from the provided map of counts. 2448 * 2449 * @param countMap The map containing the data to print. 2450 * @param heading The heading to display before printing the contents 2451 * of the map. 2452 * @param singularItem The name to use for a single item represented by the 2453 * key of the given map. 2454 * @param pluralItem The name to use for zero or multiple items 2455 * represented by the key of the given map. 2456 */ 2457 private void printCounts(@Nullable final Map<String,AtomicLong> countMap, 2458 @NotNull final String heading, 2459 @NotNull final String singularItem, 2460 @NotNull final String pluralItem) 2461 { 2462 if ((countMap == null) || countMap.isEmpty()) 2463 { 2464 return; 2465 } 2466 2467 long totalCount = 0L; 2468 for (final AtomicLong l : countMap.values()) 2469 { 2470 totalCount += l.get(); 2471 } 2472 2473 out(); 2474 out(heading); 2475 2476 int displayCount = reportCount.getValue(); 2477 if (displayCount <= 0L) 2478 { 2479 displayCount = Integer.MAX_VALUE; 2480 } 2481 2482 final List<ObjectPair<String,Long>> countList = new ArrayList<>(); 2483 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 2484 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 2485 getMostCommonElements(countMap, countList, displayCount, 2486 skippedWithSameCount, skippedWithLowerCount); 2487 2488 long count = -1L; 2489 for (final ObjectPair<String,Long> p : countList) 2490 { 2491 count = p.getSecond(); 2492 2493 if (totalCount > 0L) 2494 { 2495 final double percent = 100.0 * count / totalCount; 2496 out(p.getFirst(), ": ", count, " (", decimalFormat.format(percent), 2497 ")"); 2498 } 2499 else 2500 { 2501 out(p.getFirst(), ": ", count); 2502 } 2503 } 2504 2505 if (skippedWithSameCount.get() > 0L) 2506 { 2507 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 2508 getSingularOrPlural(skippedWithSameCount.get(), singularItem, 2509 pluralItem) + 2510 " with a count of " + count + " }"); 2511 } 2512 2513 if (skippedWithLowerCount.get() > 0L) 2514 { 2515 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 2516 getSingularOrPlural(skippedWithLowerCount.get(), singularItem, 2517 pluralItem) + 2518 " with a count that is less than " + count + " }"); 2519 } 2520 } 2521 2522 2523 2524 /** 2525 * Prints data from the provided map of counts. 2526 * 2527 * @param countMap The map containing the data to print. 2528 * @param operationType The type of operation represented by the keys of 2529 * the map. 2530 */ 2531 private void printResultCodeCounts( 2532 @Nullable final Map<ResultCode,AtomicLong> countMap, 2533 @NotNull final String operationType) 2534 { 2535 if ((countMap == null) || countMap.isEmpty()) 2536 { 2537 return; 2538 } 2539 2540 long totalCount = 0L; 2541 for (final AtomicLong l : countMap.values()) 2542 { 2543 totalCount += l.get(); 2544 } 2545 2546 out(); 2547 out("Most common " + operationType + " operation result codes:"); 2548 2549 int displayCount = reportCount.getValue(); 2550 if (displayCount <= 0L) 2551 { 2552 displayCount = Integer.MAX_VALUE; 2553 } 2554 2555 final List<ObjectPair<ResultCode,Long>> resultCodeList = new ArrayList<>(); 2556 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 2557 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 2558 getMostCommonElements(countMap, resultCodeList, displayCount, 2559 skippedWithSameCount, skippedWithLowerCount); 2560 2561 long count = -1L; 2562 for (final ObjectPair<ResultCode,Long> p : resultCodeList) 2563 { 2564 count = p.getSecond(); 2565 2566 if (totalCount > 0L) 2567 { 2568 final double percent = 100.0 * count / totalCount; 2569 out(p.getFirst().getName(), " (", p.getFirst().intValue(), "): ", 2570 count, " (", decimalFormat.format(percent), ")"); 2571 } 2572 else 2573 { 2574 out(p.getFirst(), ": ", count); 2575 } 2576 } 2577 2578 if (skippedWithSameCount.get() > 0L) 2579 { 2580 out("{ Skipped " + skippedWithSameCount.get() + " additional result " + 2581 getSingularOrPlural(skippedWithSameCount.get(), "code", "codes") + 2582 " with a count of " + count + " }"); 2583 } 2584 2585 if (skippedWithLowerCount.get() > 0L) 2586 { 2587 out("{ Skipped " + skippedWithLowerCount.get() + " additional result " + 2588 getSingularOrPlural(skippedWithLowerCount.get(), "code", "codes") + 2589 " with a count that is less than " + count + " }"); 2590 } 2591 } 2592 2593 2594 2595 /** 2596 * Retrieves the appropriate singular or plural form based on the given 2597 * value. 2598 * 2599 * @param count The count that will be used to determine whether to 2600 * retrieve the singular or plural form. 2601 * @param singular The singular form for the value to return. 2602 * @param plural The plural form for the value to return. 2603 * 2604 * @return The singular form if the count is 1, or the plural form if the 2605 * count is any other value. 2606 */ 2607 @NotNull() 2608 private String getSingularOrPlural(final long count, 2609 @NotNull final String singular, 2610 @NotNull final String plural) 2611 { 2612 if (count == 1L) 2613 { 2614 return singular; 2615 } 2616 else 2617 { 2618 return plural; 2619 } 2620 } 2621}