001 /*
002 * Copyright 2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005 /*
006 * Copyright (C) 2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021 package com.unboundid.ldap.sdk.transformations;
022
023
024
025 import java.io.ByteArrayInputStream;
026 import java.io.File;
027 import java.io.FileInputStream;
028 import java.io.FileOutputStream;
029 import java.io.InputStream;
030 import java.io.OutputStream;
031 import java.util.ArrayList;
032 import java.util.Iterator;
033 import java.util.LinkedHashMap;
034 import java.util.List;
035 import java.util.TreeMap;
036 import java.util.concurrent.atomic.AtomicLong;
037 import java.util.zip.GZIPInputStream;
038 import java.util.zip.GZIPOutputStream;
039
040 import com.unboundid.ldap.sdk.Attribute;
041 import com.unboundid.ldap.sdk.DN;
042 import com.unboundid.ldap.sdk.Entry;
043 import com.unboundid.ldap.sdk.LDAPException;
044 import com.unboundid.ldap.sdk.ResultCode;
045 import com.unboundid.ldap.sdk.Version;
046 import com.unboundid.ldap.sdk.schema.Schema;
047 import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator;
048 import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator;
049 import com.unboundid.ldif.LDIFException;
050 import com.unboundid.ldif.LDIFReader;
051 import com.unboundid.ldif.LDIFReaderChangeRecordTranslator;
052 import com.unboundid.ldif.LDIFReaderEntryTranslator;
053 import com.unboundid.ldif.LDIFRecord;
054 import com.unboundid.util.AggregateInputStream;
055 import com.unboundid.util.ByteStringBuffer;
056 import com.unboundid.util.CommandLineTool;
057 import com.unboundid.util.Debug;
058 import com.unboundid.util.StaticUtils;
059 import com.unboundid.util.ThreadSafety;
060 import com.unboundid.util.ThreadSafetyLevel;
061 import com.unboundid.util.args.ArgumentException;
062 import com.unboundid.util.args.ArgumentParser;
063 import com.unboundid.util.args.BooleanArgument;
064 import com.unboundid.util.args.DNArgument;
065 import com.unboundid.util.args.FileArgument;
066 import com.unboundid.util.args.FilterArgument;
067 import com.unboundid.util.args.IntegerArgument;
068 import com.unboundid.util.args.ScopeArgument;
069 import com.unboundid.util.args.StringArgument;
070
071 import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*;
072
073
074
075 /**
076 * This class provides a command-line tool that can be used to apply a number of
077 * transformations to an LDIF file. The transformations that can be applied
078 * include:
079 * <UL>
080 * <LI>
081 * It can scramble the values of a specified set of attributes in a manner
082 * that attempts to preserve the syntax and consistently scrambles the same
083 * value to the same representation.
084 * </LI>
085 * <LI>
086 * It can strip a specified set of attributes out of entries.
087 * </LI>
088 * <LI>
089 * It can redact the values of a specified set of attributes, to indicate
090 * that the values are there but providing no information about what their
091 * values are.
092 * </LI>
093 * <LI>
094 * It can replace the values of a specified attribute with a given set of
095 * values.
096 * </LI>
097 * <LI>
098 * It can add an attribute with a given set of values to any entry that does
099 * not contain that attribute.
100 * </LI>
101 * <LI>
102 * It can replace the values of a specified attribute with a value that
103 * contains a sequentially-incrementing counter.
104 * </LI>
105 * <LI>
106 * It can strip entries matching a given base DN, scope, and filter out of
107 * the LDIF file.
108 * </LI>
109 * <LI>
110 * It can perform DN mapping, so that entries that exist below one base DN
111 * are moved below a different base DN.
112 * </LI>
113 * <LI>
114 * It can perform attribute mapping, to replace uses of one attribute name
115 * with another.
116 * </LI>
117 * </UL>
118 */
119 @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
120 public final class TransformLDIF
121 extends CommandLineTool
122 implements LDIFReaderEntryTranslator
123 {
124 /**
125 * The maximum length of any message to write to standard output or standard
126 * error.
127 */
128 private static final int MAX_OUTPUT_LINE_LENGTH =
129 StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
130
131
132
133 // The arguments for use by this program.
134 private BooleanArgument addToExistingValues = null;
135 private BooleanArgument appendToTargetLDIF = null;
136 private BooleanArgument compressTarget = null;
137 private BooleanArgument excludeNonMatchingEntries = null;
138 private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null;
139 private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null;
140 private BooleanArgument hideRedactedValueCount = null;
141 private BooleanArgument processDNs = null;
142 private BooleanArgument sourceCompressed = null;
143 private BooleanArgument sourceContainsChangeRecords = null;
144 private BooleanArgument sourceFromStandardInput = null;
145 private BooleanArgument targetToStandardOutput = null;
146 private DNArgument addAttributeBaseDN = null;
147 private DNArgument excludeEntryBaseDN = null;
148 private DNArgument flattenBaseDN = null;
149 private DNArgument moveSubtreeFrom = null;
150 private DNArgument moveSubtreeTo = null;
151 private FileArgument schemaPath = null;
152 private FileArgument sourceLDIF = null;
153 private FileArgument targetLDIF = null;
154 private FilterArgument addAttributeFilter = null;
155 private FilterArgument excludeEntryFilter = null;
156 private FilterArgument flattenExcludeFilter = null;
157 private IntegerArgument initialSequentialValue = null;
158 private IntegerArgument numThreads = null;
159 private IntegerArgument randomSeed = null;
160 private IntegerArgument sequentialValueIncrement = null;
161 private IntegerArgument wrapColumn = null;
162 private ScopeArgument addAttributeScope = null;
163 private ScopeArgument excludeEntryScope = null;
164 private StringArgument addAttributeName = null;
165 private StringArgument addAttributeValue = null;
166 private StringArgument excludeAttribute = null;
167 private StringArgument redactAttribute = null;
168 private StringArgument renameAttributeFrom = null;
169 private StringArgument renameAttributeTo = null;
170 private StringArgument replaceValuesAttribute = null;
171 private StringArgument replacementValue = null;
172 private StringArgument scrambleAttribute = null;
173 private StringArgument scrambleJSONField = null;
174 private StringArgument sequentialAttribute = null;
175 private StringArgument textAfterSequentialValue = null;
176 private StringArgument textBeforeSequentialValue = null;
177
178 // A set of thread-local byte stream buffers that will be used to construct
179 // the LDIF representations of records.
180 private final ThreadLocal<ByteStringBuffer> byteStringBuffers =
181 new ThreadLocal<ByteStringBuffer>();
182
183
184
185 /**
186 * Invokes this tool with the provided set of arguments.
187 *
188 * @param args The command-line arguments provided to this program.
189 */
190 public static void main(final String... args)
191 {
192 final ResultCode resultCode = main(System.out, System.err, args);
193 if (resultCode != ResultCode.SUCCESS)
194 {
195 System.exit(resultCode.intValue());
196 }
197 }
198
199
200
201 /**
202 * Invokes this tool with the provided set of arguments.
203 *
204 * @param out The output stream to use for standard output. It may be
205 * {@code null} if standard output should be suppressed.
206 * @param err The output stream to use for standard error. It may be
207 * {@code null} if standard error should be suppressed.
208 * @param args The command-line arguments provided to this program.
209 *
210 * @return A result code indicating whether processing completed
211 * successfully.
212 */
213 public static ResultCode main(final OutputStream out, final OutputStream err,
214 final String... args)
215 {
216 final TransformLDIF tool = new TransformLDIF(out, err);
217 return tool.runTool(args);
218 }
219
220
221
222 /**
223 * Creates a new instance of this tool with the provided information.
224 *
225 * @param out The output stream to use for standard output. It may be
226 * {@code null} if standard output should be suppressed.
227 * @param err The output stream to use for standard error. It may be
228 * {@code null} if standard error should be suppressed.
229 */
230 public TransformLDIF(final OutputStream out, final OutputStream err)
231 {
232 super(out, err);
233 }
234
235
236
237 /**
238 * {@inheritDoc}
239 */
240 @Override()
241 public String getToolName()
242 {
243 return "transform-ldif";
244 }
245
246
247
248 /**
249 * {@inheritDoc}
250 */
251 @Override()
252 public String getToolDescription()
253 {
254 return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get();
255 }
256
257
258
259 /**
260 * {@inheritDoc}
261 */
262 @Override()
263 public String getToolVersion()
264 {
265 return Version.NUMERIC_VERSION_STRING;
266 }
267
268
269
270 /**
271 * {@inheritDoc}
272 */
273 @Override()
274 public boolean supportsInteractiveMode()
275 {
276 return true;
277 }
278
279
280
281 /**
282 * {@inheritDoc}
283 */
284 @Override()
285 public boolean defaultsToInteractiveMode()
286 {
287 return true;
288 }
289
290
291
292 /**
293 * {@inheritDoc}
294 */
295 @Override()
296 public boolean supportsPropertiesFile()
297 {
298 return true;
299 }
300
301
302
303 /**
304 * {@inheritDoc}
305 */
306 @Override()
307 public void addToolArguments(final ArgumentParser parser)
308 throws ArgumentException
309 {
310 // Add arguments pertaining to the source and target LDIF files.
311 sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null,
312 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true,
313 false);
314 sourceLDIF.addLongIdentifier("inputLDIF");
315 sourceLDIF.addLongIdentifier("source-ldif");
316 sourceLDIF.addLongIdentifier("input-ldif");
317 sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
318 parser.addArgument(sourceLDIF);
319
320 sourceFromStandardInput = new BooleanArgument(null,
321 "sourceFromStandardInput", 1,
322 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get());
323 sourceFromStandardInput.addLongIdentifier("source-from-standard-input");
324 sourceFromStandardInput.setArgumentGroupName(
325 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
326 parser.addArgument(sourceFromStandardInput);
327 parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput);
328 parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput);
329
330 targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null,
331 INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true,
332 false);
333 targetLDIF.addLongIdentifier("outputLDIF");
334 targetLDIF.addLongIdentifier("target-ldif");
335 targetLDIF.addLongIdentifier("output-ldif");
336 targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
337 parser.addArgument(targetLDIF);
338
339 targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput",
340 1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get());
341 targetToStandardOutput.addLongIdentifier("target-to-standard-output");
342 targetToStandardOutput.setArgumentGroupName(
343 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
344 parser.addArgument(targetToStandardOutput);
345 parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput);
346
347 sourceContainsChangeRecords = new BooleanArgument(null,
348 "sourceContainsChangeRecords",
349 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get());
350 sourceContainsChangeRecords.addLongIdentifier(
351 "source-contains-change-records");
352 sourceContainsChangeRecords.setArgumentGroupName(
353 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
354 parser.addArgument(sourceContainsChangeRecords);
355
356 appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF",
357 INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get());
358 appendToTargetLDIF.addLongIdentifier("append-to-target-ldif");
359 appendToTargetLDIF.setArgumentGroupName(
360 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
361 parser.addArgument(appendToTargetLDIF);
362 parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF);
363
364 wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
365 INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
366 wrapColumn.addLongIdentifier("wrap-column");
367 wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
368 parser.addArgument(wrapColumn);
369
370 sourceCompressed = new BooleanArgument('C', "sourceCompressed",
371 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get());
372 sourceCompressed.addLongIdentifier("inputCompressed");
373 sourceCompressed.addLongIdentifier("source-compressed");
374 sourceCompressed.addLongIdentifier("input-compressed");
375 sourceCompressed.setArgumentGroupName(
376 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
377 parser.addArgument(sourceCompressed);
378
379 compressTarget = new BooleanArgument('c', "compressTarget",
380 INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get());
381 compressTarget.addLongIdentifier("compressOutput");
382 compressTarget.addLongIdentifier("compress");
383 compressTarget.addLongIdentifier("compress-target");
384 compressTarget.addLongIdentifier("compress-output");
385 compressTarget.setArgumentGroupName(
386 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
387 parser.addArgument(compressTarget);
388
389
390 // Add arguments pertaining to attribute scrambling.
391 scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0,
392 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
393 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get());
394 scrambleAttribute.addLongIdentifier("attributeName");
395 scrambleAttribute.addLongIdentifier("scramble-attribute");
396 scrambleAttribute.addLongIdentifier("attribute-name");
397 scrambleAttribute.setArgumentGroupName(
398 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
399 parser.addArgument(scrambleAttribute);
400
401 scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0,
402 INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(),
403 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get(
404 scrambleAttribute.getIdentifierString()));
405 scrambleJSONField.addLongIdentifier("scramble-json-field");
406 scrambleJSONField.setArgumentGroupName(
407 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
408 parser.addArgument(scrambleJSONField);
409 parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute);
410
411 randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null,
412 INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get());
413 randomSeed.addLongIdentifier("random-seed");
414 randomSeed.setArgumentGroupName(
415 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
416 parser.addArgument(randomSeed);
417
418
419 // Add arguments pertaining to replacing attribute values with a generated
420 // value using a sequential counter.
421 sequentialAttribute = new StringArgument('S', "sequentialAttribute",
422 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
423 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get(
424 sourceContainsChangeRecords.getIdentifierString()));
425 sequentialAttribute.addLongIdentifier("sequentialAttributeName");
426 sequentialAttribute.addLongIdentifier("sequential-attribute");
427 sequentialAttribute.addLongIdentifier("sequential-attribute-name");
428 sequentialAttribute.setArgumentGroupName(
429 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
430 parser.addArgument(sequentialAttribute);
431 parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
432 sequentialAttribute);
433
434 initialSequentialValue = new IntegerArgument('i', "initialSequentialValue",
435 false, 1, null,
436 INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get(
437 sequentialAttribute.getIdentifierString()));
438 initialSequentialValue.addLongIdentifier("initial-sequential-value");
439 initialSequentialValue.setArgumentGroupName(
440 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
441 parser.addArgument(initialSequentialValue);
442 parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute);
443
444 sequentialValueIncrement = new IntegerArgument(null,
445 "sequentialValueIncrement", false, 1, null,
446 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get(
447 sequentialAttribute.getIdentifierString()));
448 sequentialValueIncrement.addLongIdentifier("sequential-value-increment");
449 sequentialValueIncrement.setArgumentGroupName(
450 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
451 parser.addArgument(sequentialValueIncrement);
452 parser.addDependentArgumentSet(sequentialValueIncrement,
453 sequentialAttribute);
454
455 textBeforeSequentialValue = new StringArgument(null,
456 "textBeforeSequentialValue", false, 1, null,
457 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get(
458 sequentialAttribute.getIdentifierString()));
459 textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value");
460 textBeforeSequentialValue.setArgumentGroupName(
461 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
462 parser.addArgument(textBeforeSequentialValue);
463 parser.addDependentArgumentSet(textBeforeSequentialValue,
464 sequentialAttribute);
465
466 textAfterSequentialValue = new StringArgument(null,
467 "textAfterSequentialValue", false, 1, null,
468 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get(
469 sequentialAttribute.getIdentifierString()));
470 textAfterSequentialValue.addLongIdentifier("text-after-sequential-value");
471 textAfterSequentialValue.setArgumentGroupName(
472 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
473 parser.addArgument(textAfterSequentialValue);
474 parser.addDependentArgumentSet(textAfterSequentialValue,
475 sequentialAttribute);
476
477
478 // Add arguments pertaining to attribute value replacement.
479 replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute",
480 false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
481 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get(
482 sourceContainsChangeRecords.getIdentifierString()));
483 replaceValuesAttribute.addLongIdentifier("replace-values-attribute");
484 replaceValuesAttribute.setArgumentGroupName(
485 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
486 parser.addArgument(replaceValuesAttribute);
487 parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
488 replaceValuesAttribute);
489
490 replacementValue = new StringArgument(null, "replacementValue", false, 0,
491 null,
492 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get(
493 replaceValuesAttribute.getIdentifierString()));
494 replacementValue.addLongIdentifier("replacement-value");
495 replacementValue.setArgumentGroupName(
496 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
497 parser.addArgument(replacementValue);
498 parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue);
499 parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute);
500
501
502 // Add arguments pertaining to adding missing attributes.
503 addAttributeName = new StringArgument(null, "addAttributeName", false, 1,
504 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
505 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get(
506 "--addAttributeValue",
507 sourceContainsChangeRecords.getIdentifierString()));
508 addAttributeName.addLongIdentifier("add-attribute-name");
509 addAttributeName.setArgumentGroupName(
510 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
511 parser.addArgument(addAttributeName);
512 parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
513 addAttributeName);
514
515 addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0,
516 null,
517 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get(
518 addAttributeName.getIdentifierString()));
519 addAttributeValue.addLongIdentifier("add-attribute-value");
520 addAttributeValue.setArgumentGroupName(
521 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
522 parser.addArgument(addAttributeValue);
523 parser.addDependentArgumentSet(addAttributeName, addAttributeValue);
524 parser.addDependentArgumentSet(addAttributeValue, addAttributeName);
525
526 addToExistingValues = new BooleanArgument(null, "addToExistingValues",
527 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get(
528 addAttributeName.getIdentifierString(),
529 addAttributeValue.getIdentifierString()));
530 addToExistingValues.addLongIdentifier("add-to-existing-values");
531 addToExistingValues.setArgumentGroupName(
532 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
533 parser.addArgument(addToExistingValues);
534 parser.addDependentArgumentSet(addToExistingValues, addAttributeName);
535
536 addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1,
537 null,
538 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get(
539 addAttributeName.getIdentifierString()));
540 addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn");
541 addAttributeBaseDN.setArgumentGroupName(
542 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
543 parser.addArgument(addAttributeBaseDN);
544 parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName);
545
546 addAttributeScope = new ScopeArgument(null, "addAttributeScope", false,
547 null,
548 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get(
549 addAttributeBaseDN.getIdentifierString(),
550 addAttributeName.getIdentifierString()));
551 addAttributeScope.addLongIdentifier("add-attribute-scope");
552 addAttributeScope.setArgumentGroupName(
553 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
554 parser.addArgument(addAttributeScope);
555 parser.addDependentArgumentSet(addAttributeScope, addAttributeName);
556
557 addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false,
558 1, null,
559 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get(
560 addAttributeName.getIdentifierString()));
561 addAttributeFilter.addLongIdentifier("add-attribute-filter");
562 addAttributeFilter.setArgumentGroupName(
563 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
564 parser.addArgument(addAttributeFilter);
565 parser.addDependentArgumentSet(addAttributeFilter, addAttributeName);
566
567
568 // Add arguments pertaining to renaming attributes.
569 renameAttributeFrom = new StringArgument(null, "renameAttributeFrom",
570 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
571 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get());
572 renameAttributeFrom.addLongIdentifier("rename-attribute-from");
573 renameAttributeFrom.setArgumentGroupName(
574 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
575 parser.addArgument(renameAttributeFrom);
576
577 renameAttributeTo = new StringArgument(null, "renameAttributeTo",
578 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
579 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get(
580 renameAttributeFrom.getIdentifierString()));
581 renameAttributeTo.addLongIdentifier("rename-attribute-to");
582 renameAttributeTo.setArgumentGroupName(
583 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
584 parser.addArgument(renameAttributeTo);
585 parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo);
586 parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom);
587
588
589 // Add arguments pertaining to flattening subtrees.
590 flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null,
591 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get());
592 flattenBaseDN.addLongIdentifier("flatten-base-dn");
593 flattenBaseDN.setArgumentGroupName(
594 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
595 parser.addArgument(flattenBaseDN);
596 parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
597 flattenBaseDN);
598
599 flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null,
600 "flattenAddOmittedRDNAttributesToEntry", 1,
601 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get());
602 flattenAddOmittedRDNAttributesToEntry.addLongIdentifier(
603 "flatten-add-omitted-rdn-attributes-to-entry");
604 flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName(
605 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
606 parser.addArgument(flattenAddOmittedRDNAttributesToEntry);
607 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry,
608 flattenBaseDN);
609
610 flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null,
611 "flattenAddOmittedRDNAttributesToRDN", 1,
612 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get());
613 flattenAddOmittedRDNAttributesToRDN.addLongIdentifier(
614 "flatten-add-omitted-rdn-attributes-to-rdn");
615 flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName(
616 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
617 parser.addArgument(flattenAddOmittedRDNAttributesToRDN);
618 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN,
619 flattenBaseDN);
620
621 flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter",
622 false, 1, null,
623 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get());
624 flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter");
625 flattenExcludeFilter.setArgumentGroupName(
626 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
627 parser.addArgument(flattenExcludeFilter);
628 parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN);
629
630
631 // Add arguments pertaining to moving subtrees.
632 moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null,
633 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get());
634 moveSubtreeFrom.addLongIdentifier("move-subtree-from");
635 moveSubtreeFrom.setArgumentGroupName(
636 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
637 parser.addArgument(moveSubtreeFrom);
638
639 moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null,
640 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get(
641 moveSubtreeFrom.getIdentifierString()));
642 moveSubtreeTo.addLongIdentifier("move-subtree-to");
643 moveSubtreeTo.setArgumentGroupName(
644 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
645 parser.addArgument(moveSubtreeTo);
646 parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo);
647 parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom);
648
649
650 // Add arguments pertaining to redacting attribute values.
651 redactAttribute = new StringArgument(null, "redactAttribute", false, 0,
652 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
653 INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get());
654 redactAttribute.addLongIdentifier("redact-attribute");
655 redactAttribute.setArgumentGroupName(
656 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
657 parser.addArgument(redactAttribute);
658
659 hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount",
660 INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get());
661 hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count");
662 hideRedactedValueCount.setArgumentGroupName(
663 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
664 parser.addArgument(hideRedactedValueCount);
665 parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute);
666
667
668 // Add arguments pertaining to excluding attributes and entries.
669 excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0,
670 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
671 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get());
672 excludeAttribute.addLongIdentifier("suppressAttribute");
673 excludeAttribute.addLongIdentifier("exclude-attribute");
674 excludeAttribute.addLongIdentifier("suppress-attribute");
675 excludeAttribute.setArgumentGroupName(
676 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
677 parser.addArgument(excludeAttribute);
678
679 excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1,
680 null,
681 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get(
682 sourceContainsChangeRecords.getIdentifierString()));
683 excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN");
684 excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn");
685 excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn");
686 excludeEntryBaseDN.setArgumentGroupName(
687 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
688 parser.addArgument(excludeEntryBaseDN);
689 parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
690 excludeEntryBaseDN);
691
692 excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false,
693 null,
694 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get(
695 sourceContainsChangeRecords.getIdentifierString()));
696 excludeEntryScope.addLongIdentifier("suppressEntryScope");
697 excludeEntryScope.addLongIdentifier("exclude-entry-scope");
698 excludeEntryScope.addLongIdentifier("suppress-entry-scope");
699 excludeEntryScope.setArgumentGroupName(
700 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
701 parser.addArgument(excludeEntryScope);
702 parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
703 excludeEntryScope);
704
705 excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false,
706 1, null,
707 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get(
708 sourceContainsChangeRecords.getIdentifierString()));
709 excludeEntryFilter.addLongIdentifier("suppressEntryFilter");
710 excludeEntryFilter.addLongIdentifier("exclude-entry-filter");
711 excludeEntryFilter.addLongIdentifier("suppress-entry-filter");
712 excludeEntryFilter.setArgumentGroupName(
713 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
714 parser.addArgument(excludeEntryFilter);
715 parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
716 excludeEntryFilter);
717
718 excludeNonMatchingEntries = new BooleanArgument(null,
719 "excludeNonMatchingEntries",
720 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get());
721 excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries");
722 excludeNonMatchingEntries.setArgumentGroupName(
723 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
724 parser.addArgument(excludeNonMatchingEntries);
725 parser.addDependentArgumentSet(excludeNonMatchingEntries,
726 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
727
728
729 // Add the remaining arguments.
730 schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
731 INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(),
732 true, true, false, false);
733 schemaPath.addLongIdentifier("schemaFile");
734 schemaPath.addLongIdentifier("schemaDirectory");
735 schemaPath.addLongIdentifier("schema-path");
736 schemaPath.addLongIdentifier("schema-file");
737 schemaPath.addLongIdentifier("schema-directory");
738 parser.addArgument(schemaPath);
739
740 numThreads = new IntegerArgument('t', "numThreads", false, 1, null,
741 INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE,
742 1);
743 numThreads.addLongIdentifier("num-threads");
744 parser.addArgument(numThreads);
745
746 processDNs = new BooleanArgument('d', "processDNs",
747 INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get());
748 processDNs.addLongIdentifier("process-dns");
749 parser.addArgument(processDNs);
750
751
752 // Ensure that at least one kind of transformation was requested.
753 parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute,
754 replaceValuesAttribute, addAttributeName, renameAttributeFrom,
755 flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute,
756 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
757 }
758
759
760
761 /**
762 * {@inheritDoc}
763 */
764 @Override()
765 public void doExtendedArgumentValidation()
766 throws ArgumentException
767 {
768 // Ideally, exactly one of the targetLDIF and targetToStandardOutput
769 // arguments should always be provided. But in order to preserve backward
770 // compatibility with a legacy scramble-ldif tool, we will allow both to be
771 // omitted if either --scrambleAttribute or --sequentialArgument is
772 // provided. In that case, the path of the output file will be the path of
773 // the first input file with ".scrambled" appended to it.
774 if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent()))
775 {
776 if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent()))
777 {
778 throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get(
779 targetLDIF.getIdentifierString(),
780 targetToStandardOutput.getIdentifierString()));
781 }
782 }
783
784
785 // Make sure that the --renameAttributeFrom and --renameAttributeTo
786 // arguments were provided an equal number of times.
787 final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences();
788 final int renameToOccurrences = renameAttributeTo.getNumOccurrences();
789 if (renameFromOccurrences != renameToOccurrences)
790 {
791 throw new ArgumentException(
792 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
793 renameAttributeFrom.getIdentifierString(),
794 renameAttributeTo.getIdentifierString()));
795 }
796
797
798 // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were
799 // provided an equal number of times.
800 final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences();
801 final int moveToOccurrences = moveSubtreeTo.getNumOccurrences();
802 if (moveFromOccurrences != moveToOccurrences)
803 {
804 throw new ArgumentException(
805 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
806 moveSubtreeFrom.getIdentifierString(),
807 moveSubtreeTo.getIdentifierString()));
808 }
809 }
810
811
812
813 /**
814 * {@inheritDoc}
815 */
816 @Override()
817 public ResultCode doToolProcessing()
818 {
819 final Schema schema;
820 try
821 {
822 schema = getSchema();
823 }
824 catch (final LDAPException le)
825 {
826 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage());
827 return le.getResultCode();
828 }
829
830
831 // Create the translators to use to apply the transformations.
832 final ArrayList<LDIFReaderEntryTranslator> entryTranslators =
833 new ArrayList<LDIFReaderEntryTranslator>(10);
834 final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators =
835 new ArrayList<LDIFReaderChangeRecordTranslator>(10);
836
837 final AtomicLong excludedEntryCount = new AtomicLong(0L);
838 createTranslators(entryTranslators, changeRecordTranslators,
839 schema, excludedEntryCount);
840
841 final AggregateLDIFReaderEntryTranslator entryTranslator =
842 new AggregateLDIFReaderEntryTranslator(entryTranslators);
843 final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator =
844 new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators);
845
846
847 // Determine the path to the target file to be written.
848 final File targetFile;
849 if (targetLDIF.isPresent())
850 {
851 targetFile = targetLDIF.getValue();
852 }
853 else if (targetToStandardOutput.isPresent())
854 {
855 targetFile = null;
856 }
857 else
858 {
859 targetFile =
860 new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled");
861 }
862
863
864 // Create the LDIF reader.
865 final LDIFReader ldifReader;
866 try
867 {
868 InputStream inputStream;
869 if (sourceLDIF.isPresent())
870 {
871 final List<File> sourceFiles = sourceLDIF.getValues();
872 final ArrayList<InputStream> fileInputStreams =
873 new ArrayList<InputStream>(2*sourceFiles.size());
874 for (final File f : sourceFiles)
875 {
876 if (! fileInputStreams.isEmpty())
877 {
878 // Go ahead and ensure that there are at least new end-of-line
879 // markers between each file. Otherwise, it's possible for entries
880 // to run together.
881 final byte[] doubleEOL = new byte[StaticUtils.EOL_BYTES.length * 2];
882 System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL, 0,
883 StaticUtils.EOL_BYTES.length);
884 System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL,
885 StaticUtils.EOL_BYTES.length, StaticUtils.EOL_BYTES.length);
886 fileInputStreams.add(new ByteArrayInputStream(doubleEOL));
887 }
888 fileInputStreams.add(new FileInputStream(f));
889 }
890
891 if (fileInputStreams.size() == 1)
892 {
893 inputStream = fileInputStreams.get(0);
894 }
895 else
896 {
897 inputStream = new AggregateInputStream(fileInputStreams);
898 }
899 }
900 else
901 {
902 inputStream = System.in;
903 }
904
905 if (sourceCompressed.isPresent())
906 {
907 inputStream = new GZIPInputStream(inputStream);
908 }
909
910 ldifReader = new LDIFReader(inputStream, numThreads.getValue(),
911 entryTranslator, changeRecordTranslator);
912 if (schema != null)
913 {
914 ldifReader.setSchema(schema);
915 }
916 }
917 catch (final Exception e)
918 {
919 Debug.debugException(e);
920 wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
921 ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get(
922 StaticUtils.getExceptionMessage(e)));
923 return ResultCode.LOCAL_ERROR;
924 }
925
926
927 ResultCode resultCode = ResultCode.SUCCESS;
928 OutputStream outputStream = null;
929 processingBlock:
930 try
931 {
932 // Create the output stream to use to write the transformed data.
933 try
934 {
935 if (targetFile == null)
936 {
937 outputStream = getOut();
938 }
939 else
940 {
941 outputStream =
942 new FileOutputStream(targetFile, appendToTargetLDIF.isPresent());
943 }
944
945 if (compressTarget.isPresent())
946 {
947 outputStream = new GZIPOutputStream(outputStream);
948 }
949 }
950 catch (final Exception e)
951 {
952 Debug.debugException(e);
953 wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
954 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get(
955 targetFile.getAbsolutePath(),
956 StaticUtils.getExceptionMessage(e)));
957 resultCode = ResultCode.LOCAL_ERROR;
958 break processingBlock;
959 }
960
961
962 // Read the source data one record at a time. The transformations will
963 // automatically be applied by the LDIF reader's translators, and even if
964 // there are multiple reader threads, we're guaranteed to get the results
965 // in the right order.
966 long entriesWritten = 0L;
967 while (true)
968 {
969 final LDIFRecord ldifRecord;
970 try
971 {
972 ldifRecord = ldifReader.readLDIFRecord();
973 }
974 catch (final LDIFException le)
975 {
976 Debug.debugException(le);
977 if (le.mayContinueReading())
978 {
979 wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
980 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get(
981 StaticUtils.getExceptionMessage(le)));
982 if (resultCode == ResultCode.SUCCESS)
983 {
984 resultCode = ResultCode.PARAM_ERROR;
985 }
986 continue;
987 }
988 else
989 {
990 wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
991 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get(
992 StaticUtils.getExceptionMessage(le)));
993 if (resultCode == ResultCode.SUCCESS)
994 {
995 resultCode = ResultCode.PARAM_ERROR;
996 }
997 break processingBlock;
998 }
999 }
1000 catch (final Exception e)
1001 {
1002 Debug.debugException(e);
1003 wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1004 ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get(
1005 StaticUtils.getExceptionMessage(e)));
1006 resultCode = ResultCode.LOCAL_ERROR;
1007 break processingBlock;
1008 }
1009
1010
1011 // If the LDIF record is null, then we've run out of records so we're
1012 // done.
1013 if (ldifRecord == null)
1014 {
1015 break;
1016 }
1017
1018
1019 // Write the record to the output stream.
1020 try
1021 {
1022 if (ldifRecord instanceof PreEncodedLDIFEntry)
1023 {
1024 outputStream.write(
1025 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes());
1026 }
1027 else
1028 {
1029 final ByteStringBuffer buffer = getBuffer();
1030 if (wrapColumn.isPresent())
1031 {
1032 ldifRecord.toLDIF(buffer, wrapColumn.getValue());
1033 }
1034 else
1035 {
1036 ldifRecord.toLDIF(buffer, 0);
1037 }
1038 buffer.append(StaticUtils.EOL_BYTES);
1039 buffer.write(outputStream);
1040 }
1041 }
1042 catch (final Exception e)
1043 {
1044 Debug.debugException(e);
1045 wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1046 ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(),
1047 StaticUtils.getExceptionMessage(e)));
1048 resultCode = ResultCode.LOCAL_ERROR;
1049 break processingBlock;
1050 }
1051
1052
1053 // If we've written a multiple of 1000 entries, print a progress
1054 // message.
1055 entriesWritten++;
1056 if ((! targetToStandardOutput.isPresent()) &&
1057 ((entriesWritten % 1000L) == 0))
1058 {
1059 final long numExcluded = excludedEntryCount.get();
1060 if (numExcluded > 0L)
1061 {
1062 wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1063 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get(
1064 entriesWritten, numExcluded));
1065 }
1066 else
1067 {
1068 wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1069 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get(
1070 entriesWritten));
1071 }
1072 }
1073 }
1074
1075
1076 if (! targetToStandardOutput.isPresent())
1077 {
1078 final long numExcluded = excludedEntryCount.get();
1079 if (numExcluded > 0L)
1080 {
1081 wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1082 INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten,
1083 numExcluded));
1084 }
1085 else
1086 {
1087 wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1088 INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten));
1089 }
1090 }
1091 }
1092 finally
1093 {
1094 if (outputStream != null)
1095 {
1096 try
1097 {
1098 outputStream.close();
1099 }
1100 catch (final Exception e)
1101 {
1102 Debug.debugException(e);
1103 wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1104 ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get(
1105 targetFile.getAbsolutePath(),
1106 StaticUtils.getExceptionMessage(e)));
1107 if (resultCode == ResultCode.SUCCESS)
1108 {
1109 resultCode = ResultCode.LOCAL_ERROR;
1110 }
1111 }
1112 }
1113
1114 try
1115 {
1116 ldifReader.close();
1117 }
1118 catch (final Exception e)
1119 {
1120 Debug.debugException(e);
1121 // We can ignore this.
1122 }
1123 }
1124
1125
1126 return resultCode;
1127 }
1128
1129
1130
1131 /**
1132 * Retrieves the schema that should be used for processing.
1133 *
1134 * @return The schema that was created.
1135 *
1136 * @throws LDAPException If a problem is encountered while retrieving the
1137 * schema.
1138 */
1139 private Schema getSchema()
1140 throws LDAPException
1141 {
1142 // If any schema paths were specified, then load the schema only from those
1143 // paths.
1144 if (schemaPath.isPresent())
1145 {
1146 final ArrayList<File> schemaFiles = new ArrayList<File>(10);
1147 for (final File path : schemaPath.getValues())
1148 {
1149 if (path.isFile())
1150 {
1151 schemaFiles.add(path);
1152 }
1153 else
1154 {
1155 final TreeMap<String,File> fileMap = new TreeMap<String,File>();
1156 for (final File schemaDirFile : path.listFiles())
1157 {
1158 final String name = schemaDirFile.getName();
1159 if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif"))
1160 {
1161 fileMap.put(name, schemaDirFile);
1162 }
1163 }
1164 schemaFiles.addAll(fileMap.values());
1165 }
1166 }
1167
1168 if (schemaFiles.isEmpty())
1169 {
1170 throw new LDAPException(ResultCode.PARAM_ERROR,
1171 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get(
1172 schemaPath.getIdentifierString()));
1173 }
1174 else
1175 {
1176 try
1177 {
1178 return Schema.getSchema(schemaFiles);
1179 }
1180 catch (final Exception e)
1181 {
1182 Debug.debugException(e);
1183 throw new LDAPException(ResultCode.LOCAL_ERROR,
1184 ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get(
1185 StaticUtils.getExceptionMessage(e)));
1186 }
1187 }
1188 }
1189 else
1190 {
1191 // If the INSTANCE_ROOT environment variable is set and it refers to a
1192 // directory that has a config/schema subdirectory that has one or more
1193 // schema files in it, then read the schema from that directory.
1194 try
1195 {
1196 final String instanceRootStr = System.getenv("INSTANCE_ROOT");
1197 if (instanceRootStr != null)
1198 {
1199 final File instanceRoot = new File(instanceRootStr);
1200 final File configDir = new File(instanceRoot, "config");
1201 final File schemaDir = new File(configDir, "schema");
1202 if (schemaDir.exists())
1203 {
1204 final TreeMap<String,File> fileMap = new TreeMap<String,File>();
1205 for (final File schemaDirFile : schemaDir.listFiles())
1206 {
1207 final String name = schemaDirFile.getName();
1208 if (schemaDirFile.isFile() &&
1209 name.toLowerCase().endsWith(".ldif"))
1210 {
1211 fileMap.put(name, schemaDirFile);
1212 }
1213 }
1214
1215 if (! fileMap.isEmpty())
1216 {
1217 return Schema.getSchema(new ArrayList<File>(fileMap.values()));
1218 }
1219 }
1220 }
1221 }
1222 catch (final Exception e)
1223 {
1224 Debug.debugException(e);
1225 }
1226 }
1227
1228
1229 // If we've gotten here, then just return null and the tool will try to use
1230 // the default standard schema.
1231 return null;
1232 }
1233
1234
1235
1236 /**
1237 * Creates the entry and change record translators that will be used to
1238 * perform the transformations.
1239 *
1240 * @param entryTranslators A list to which all created entry
1241 * translators should be written.
1242 * @param changeRecordTranslators A list to which all created change record
1243 * translators should be written.
1244 * @param schema The schema to use when processing.
1245 * @param excludedEntryCount A counter used to keep track of the number
1246 * of entries that have been excluded from
1247 * the result set.
1248 */
1249 private void createTranslators(
1250 final List<LDIFReaderEntryTranslator> entryTranslators,
1251 final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators,
1252 final Schema schema, final AtomicLong excludedEntryCount)
1253 {
1254 if (scrambleAttribute.isPresent())
1255 {
1256 final Long seed;
1257 if (randomSeed.isPresent())
1258 {
1259 seed = randomSeed.getValue().longValue();
1260 }
1261 else
1262 {
1263 seed = null;
1264 }
1265
1266 final ScrambleAttributeTransformation t =
1267 new ScrambleAttributeTransformation(schema, seed,
1268 processDNs.isPresent(), scrambleAttribute.getValues(),
1269 scrambleJSONField.getValues());
1270 entryTranslators.add(t);
1271 changeRecordTranslators.add(t);
1272 }
1273
1274 if (sequentialAttribute.isPresent())
1275 {
1276 final long initialValue;
1277 if (initialSequentialValue.isPresent())
1278 {
1279 initialValue = initialSequentialValue.getValue().longValue();
1280 }
1281 else
1282 {
1283 initialValue = 0L;
1284 }
1285
1286 final long incrementAmount;
1287 if (sequentialValueIncrement.isPresent())
1288 {
1289 incrementAmount = sequentialValueIncrement.getValue().longValue();
1290 }
1291 else
1292 {
1293 incrementAmount = 1L;
1294 }
1295
1296 for (final String attrName : sequentialAttribute.getValues())
1297 {
1298
1299
1300 final ReplaceWithCounterTransformation t =
1301 new ReplaceWithCounterTransformation(schema, attrName,
1302 initialValue, incrementAmount,
1303 textBeforeSequentialValue.getValue(),
1304 textAfterSequentialValue.getValue(), processDNs.isPresent());
1305 entryTranslators.add(t);
1306 }
1307 }
1308
1309 if (replaceValuesAttribute.isPresent())
1310 {
1311 final ReplaceAttributeTransformation t =
1312 new ReplaceAttributeTransformation(schema,
1313 replaceValuesAttribute.getValue(),
1314 replacementValue.getValues());
1315 entryTranslators.add(t);
1316 }
1317
1318 if (addAttributeName.isPresent())
1319 {
1320 final AddAttributeTransformation t = new AddAttributeTransformation(
1321 schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(),
1322 addAttributeFilter.getValue(),
1323 new Attribute(addAttributeName.getValue(), schema,
1324 addAttributeValue.getValues()),
1325 (! addToExistingValues.isPresent()));
1326 entryTranslators.add(t);
1327 }
1328
1329 if (renameAttributeFrom.isPresent())
1330 {
1331 final Iterator<String> renameFromIterator =
1332 renameAttributeFrom.getValues().iterator();
1333 final Iterator<String> renameToIterator =
1334 renameAttributeTo.getValues().iterator();
1335 while (renameFromIterator.hasNext())
1336 {
1337 final RenameAttributeTransformation t =
1338 new RenameAttributeTransformation(schema,
1339 renameFromIterator.next(), renameToIterator.next(),
1340 processDNs.isPresent());
1341 entryTranslators.add(t);
1342 changeRecordTranslators.add(t);
1343 }
1344 }
1345
1346 if (flattenBaseDN.isPresent())
1347 {
1348 final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation(
1349 schema, flattenBaseDN.getValue(),
1350 flattenAddOmittedRDNAttributesToEntry.isPresent(),
1351 flattenAddOmittedRDNAttributesToRDN.isPresent(),
1352 flattenExcludeFilter.getValue());
1353 entryTranslators.add(t);
1354 }
1355
1356 if (moveSubtreeFrom.isPresent())
1357 {
1358 final Iterator<DN> moveFromIterator =
1359 moveSubtreeFrom.getValues().iterator();
1360 final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator();
1361 while (moveFromIterator.hasNext())
1362 {
1363 final MoveSubtreeTransformation t =
1364 new MoveSubtreeTransformation(moveFromIterator.next(),
1365 moveToIterator.next());
1366 entryTranslators.add(t);
1367 changeRecordTranslators.add(t);
1368 }
1369 }
1370
1371 if (redactAttribute.isPresent())
1372 {
1373 final RedactAttributeTransformation t = new RedactAttributeTransformation(
1374 schema, processDNs.isPresent(),
1375 (! hideRedactedValueCount.isPresent()), redactAttribute.getValues());
1376 entryTranslators.add(t);
1377 changeRecordTranslators.add(t);
1378 }
1379
1380 if (excludeAttribute.isPresent())
1381 {
1382 final ExcludeAttributeTransformation t =
1383 new ExcludeAttributeTransformation(schema,
1384 excludeAttribute.getValues());
1385 entryTranslators.add(t);
1386 changeRecordTranslators.add(t);
1387 }
1388
1389 if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() ||
1390 excludeEntryFilter.isPresent())
1391 {
1392 final ExcludeEntryTransformation t = new ExcludeEntryTransformation(
1393 schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(),
1394 excludeEntryFilter.getValue(),
1395 (! excludeNonMatchingEntries.isPresent()), excludedEntryCount);
1396 entryTranslators.add(t);
1397 }
1398
1399 entryTranslators.add(this);
1400 }
1401
1402
1403
1404 /**
1405 * {@inheritDoc}
1406 */
1407 @Override()
1408 public LinkedHashMap<String[],String> getExampleUsages()
1409 {
1410 final LinkedHashMap<String[],String> examples =
1411 new LinkedHashMap<String[],String>(4);
1412
1413 examples.put(
1414 new String[]
1415 {
1416 "--sourceLDIF", "input.ldif",
1417 "--targetLDIF", "scrambled.ldif",
1418 "--scrambleAttribute", "givenName",
1419 "--scrambleAttribute", "sn",
1420 "--scrambleAttribute", "cn",
1421 "--numThreads", "10",
1422 "--schemaPath", "/ds/UnboundID-DS/config/schema",
1423 "--processDNs"
1424 },
1425 INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get());
1426
1427 examples.put(
1428 new String[]
1429 {
1430 "--sourceLDIF", "input.ldif",
1431 "--targetLDIF", "sequential.ldif",
1432 "--sequentialAttribute", "uid",
1433 "--initialSequentialValue", "1",
1434 "--sequentialValueIncrement", "1",
1435 "--textBeforeSequentialValue", "user.",
1436 "--numThreads", "10",
1437 "--schemaPath", "/ds/UnboundID-DS/config/schema",
1438 "--processDNs"
1439 },
1440 INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get());
1441
1442 examples.put(
1443 new String[]
1444 {
1445 "--sourceLDIF", "input.ldif",
1446 "--targetLDIF", "added-organization.ldif",
1447 "--addAttributeName", "o",
1448 "--addAttributeValue", "Example Corp.",
1449 "--addAttributeFilter", "(objectClass=person)",
1450 "--numThreads", "10",
1451 "--schemaPath", "/ds/UnboundID-DS/config/schema"
1452 },
1453 INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get());
1454
1455 examples.put(
1456 new String[]
1457 {
1458 "--sourceLDIF", "input.ldif",
1459 "--targetLDIF", "rebased.ldif",
1460 "--moveSubtreeFrom", "o=example.com",
1461 "--moveSubtreeTo", "dc=example,dc=com",
1462 "--numThreads", "10",
1463 "--schemaPath", "/ds/UnboundID-DS/config/schema"
1464 },
1465 INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get());
1466
1467 return examples;
1468 }
1469
1470
1471
1472 /**
1473 * {@inheritDoc}
1474 */
1475 public Entry translate(final Entry original, final long firstLineNumber)
1476 throws LDIFException
1477 {
1478 final ByteStringBuffer buffer = getBuffer();
1479 if (wrapColumn.isPresent())
1480 {
1481 original.toLDIF(buffer, wrapColumn.getValue());
1482 }
1483 else
1484 {
1485 original.toLDIF(buffer, 0);
1486 }
1487 buffer.append(StaticUtils.EOL_BYTES);
1488
1489 return new PreEncodedLDIFEntry(original, buffer.toByteArray());
1490 }
1491
1492
1493
1494 /**
1495 * Retrieves a byte string buffer that can be used to perform LDIF encoding.
1496 *
1497 * @return A byte string buffer that can be used to perform LDIF encoding.
1498 */
1499 private ByteStringBuffer getBuffer()
1500 {
1501 ByteStringBuffer buffer = byteStringBuffers.get();
1502 if (buffer == null)
1503 {
1504 buffer = new ByteStringBuffer();
1505 byteStringBuffers.set(buffer);
1506 }
1507 else
1508 {
1509 buffer.clear();
1510 }
1511
1512 return buffer;
1513 }
1514 }