001/*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License").  You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
010 * or http://forgerock.org/license/CDDLv1.0.html.
011 * See the License for the specific language governing permissions
012 * and limitations under the License.
013 *
014 * When distributing Covered Code, include this CDDL HEADER in each
015 * file and include the License file at legal-notices/CDDLv1_0.txt.
016 * If applicable, add the following below this CDDL HEADER, with the
017 * fields enclosed by brackets "[]" replaced with your own identifying
018 * information:
019 *      Portions Copyright [yyyy] [name of copyright owner]
020 *
021 * CDDL HEADER END
022 *
023 *
024 *      Copyright 2006-2010 Sun Microsystems, Inc.
025 *      Portions Copyright 2011-2015 ForgeRock AS.
026 */
027package org.opends.server.tools;
028
029import static org.opends.messages.ToolMessages.*;
030import static org.opends.server.config.ConfigConstants.*;
031import static org.opends.server.util.StaticUtils.*;
032
033import static com.forgerock.opendj.cli.ArgumentConstants.*;
034import static com.forgerock.opendj.cli.Utils.*;
035
036import java.io.File;
037import java.io.OutputStream;
038import java.io.PrintStream;
039import java.util.ArrayList;
040import java.util.HashSet;
041import java.util.List;
042import java.util.Random;
043import java.util.Set;
044
045import org.forgerock.i18n.LocalizableMessage;
046import org.forgerock.i18n.slf4j.LocalizedLogger;
047import org.forgerock.opendj.config.server.ConfigException;
048import org.forgerock.opendj.ldap.ResultCode;
049import org.opends.server.admin.std.server.BackendCfg;
050import org.opends.server.api.Backend;
051import org.opends.server.api.Backend.BackendOperation;
052import org.opends.server.api.plugin.PluginType;
053import org.opends.server.core.CoreConfigManager;
054import org.opends.server.core.DirectoryServer;
055import org.opends.server.core.LockFileManager;
056import org.opends.server.core.PluginConfigManager;
057import org.opends.server.extensions.ConfigFileHandler;
058import org.opends.server.loggers.ErrorLogPublisher;
059import org.opends.server.loggers.ErrorLogger;
060import org.opends.server.loggers.JDKLogging;
061import org.opends.server.loggers.TextErrorLogPublisher;
062import org.opends.server.loggers.TextWriter;
063import org.opends.server.protocols.ldap.LDAPAttribute;
064import org.opends.server.tasks.ImportTask;
065import org.opends.server.tools.makeldif.TemplateFile;
066import org.opends.server.tools.tasks.TaskTool;
067import org.opends.server.types.AttributeType;
068import org.opends.server.types.DN;
069import org.opends.server.types.DirectoryException;
070import org.opends.server.types.ExistingFileBehavior;
071import org.opends.server.types.InitializationException;
072import org.opends.server.types.LDIFImportConfig;
073import org.opends.server.types.LDIFImportResult;
074import org.opends.server.types.NullOutputStream;
075import org.opends.server.types.RawAttribute;
076import org.opends.server.types.SearchFilter;
077import org.opends.server.util.args.LDAPConnectionArgumentParser;
078
079import com.forgerock.opendj.cli.Argument;
080import com.forgerock.opendj.cli.ArgumentException;
081import com.forgerock.opendj.cli.BooleanArgument;
082import com.forgerock.opendj.cli.ClientException;
083import com.forgerock.opendj.cli.CommonArguments;
084import com.forgerock.opendj.cli.IntegerArgument;
085import com.forgerock.opendj.cli.StringArgument;
086
087/**
088 * This program provides a utility that may be used to import the contents of an
089 * LDIF file into a Directory Server backend.  This will be a process that is
090 * intended to run separate from Directory Server and not internally within the
091 * server process (e.g., via the tasks interface).
092 */
093public class ImportLDIF extends TaskTool {
094
095  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
096
097  /**
098   * The buffer size that should be used when reading data from LDIF.
099   */
100  public static final int LDIF_BUFFER_SIZE = 1048576;
101
102
103  /**
104   * The main method for ImportLDIF tool.
105   *
106   * @param  args  The command-line arguments provided to this program.
107   */
108  public static void main(String[] args)
109  {
110    int retCode = mainImportLDIF(args, true, System.out, System.err);
111
112    if(retCode != 0)
113    {
114      System.exit(filterExitCode(retCode));
115    }
116  }
117
118  /**
119   * Processes the command-line arguments and invokes the import process.
120   *
121   * @param  args  The command-line arguments provided to this program.
122   * @return The error code.
123   */
124  public static int mainImportLDIF(String[] args)
125  {
126    return mainImportLDIF(args, true, System.out, System.err);
127  }
128
129  /**
130   * Processes the command-line arguments and invokes the import process.
131   *
132   * @param  args              The command-line arguments provided to this
133   *                           program.
134   * @param  initializeServer  Indicates whether to initialize the server.
135   * @param  outStream         The output stream to use for standard output, or
136   *                           {@code null} if standard output is not needed.
137   * @param  errStream         The output stream to use for standard error, or
138   *                           {@code null} if standard error is not needed.
139   *
140   * @return The error code.
141   */
142  public static int mainImportLDIF(String[] args, boolean initializeServer,
143                                   OutputStream outStream,
144                                   OutputStream errStream)
145  {
146    ImportLDIF tool = new ImportLDIF();
147    return tool.process(args, initializeServer, outStream, errStream);
148  }
149
150  /** Define the command-line arguments that may be used with this program. */
151  private BooleanArgument append;
152  private BooleanArgument countRejects;
153  private BooleanArgument displayUsage;
154  private BooleanArgument isCompressed;
155  private BooleanArgument isEncrypted;
156  private BooleanArgument overwrite;
157  private BooleanArgument quietMode;
158  private BooleanArgument replaceExisting;
159  private BooleanArgument skipSchemaValidation;
160  private BooleanArgument clearBackend;
161  private IntegerArgument randomSeed;
162  private StringArgument  backendID;
163  private StringArgument  configClass;
164  private StringArgument  configFile;
165  private StringArgument  excludeAttributeStrings;
166  private StringArgument  excludeBranchStrings;
167  private StringArgument  excludeFilterStrings;
168  private StringArgument  includeAttributeStrings;
169  private StringArgument  includeBranchStrings;
170  private StringArgument  includeFilterStrings;
171  private StringArgument  ldifFiles;
172  private StringArgument  rejectFile;
173  private StringArgument  skipFile;
174  private StringArgument  templateFile;
175  private BooleanArgument skipDNValidation;
176  private IntegerArgument threadCount;
177  private StringArgument  tmpDirectory;
178
179  private int process(String[] args, boolean initializeServer,
180                      OutputStream outStream, OutputStream errStream) {
181
182    PrintStream out = NullOutputStream.wrapOrNullStream(outStream);
183    PrintStream err = NullOutputStream.wrapOrNullStream(errStream);
184    JDKLogging.disableLogging();
185
186    // FIXME -- Need to add a mechanism for verifying the file signature.
187
188
189    // Create the command-line argument parser for use with this program.
190    LDAPConnectionArgumentParser argParser =
191            createArgParser("org.opends.server.tools.ImportLDIF",
192                            INFO_LDIFIMPORT_TOOL_DESCRIPTION.get());
193    argParser.setShortToolDescription(REF_SHORT_DESC_IMPORT_LDIF.get());
194
195    // Initialize all the command-line argument types and register them with the
196    // parser.
197    try
198    {
199      createArguments(argParser);
200    }
201    catch (ArgumentException ae)
202    {
203      printWrappedText(err, ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage()));
204      return 1;
205    }
206
207    // Init the default values so that they can appear also on the usage.
208    try
209    {
210      argParser.getArguments().initArgumentsWithConfiguration();
211    }
212    catch (ConfigException ce)
213    {
214      // Ignore.
215    }
216
217    // Parse the command-line arguments provided to this program.
218    try
219    {
220      argParser.parseArguments(args);
221      validateTaskArgs();
222    }
223    catch (ArgumentException ae)
224    {
225      argParser.displayMessageAndUsageReference(err, ERR_ERROR_PARSING_ARGS.get(ae.getMessage()));
226      return 1;
227    }
228    catch (ClientException ce)
229    {
230      // No need to display the usage since the problem comes with a provided value.
231      printWrappedText(err, ce.getMessageObject());
232      return 1;
233    }
234
235
236    if (argParser.usageOrVersionDisplayed())
237    {
238      return 0;
239    }
240
241
242    // Make sure that either the "ldifFile" argument or the "templateFile"
243    // argument was provided, but not both.
244    if (ldifFiles.isPresent())
245    {
246      if (templateFile.isPresent())
247      {
248        printWrappedText(err,
249            ERR_LDIFIMPORT_CONFLICTING_OPTIONS.get(ldifFiles.getLongIdentifier(), templateFile.getLongIdentifier()));
250        return 1;
251      }
252    }
253    else if (! templateFile.isPresent())
254    {
255      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_REQUIRED_ARGUMENT.get(
256          ldifFiles.getLongIdentifier(), templateFile.getLongIdentifier()));
257      return 1;
258    }
259
260    // Make sure that either the "includeBranchStrings" argument or the
261    // "backendID" argument was provided.
262    if(!includeBranchStrings.isPresent() && !backendID.isPresent())
263    {
264      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_BACKEND_ARGUMENT.get(
265          includeBranchStrings.getLongIdentifier(), backendID.getLongIdentifier()));
266      return 1;
267    }
268
269    // Count rejects option requires the ability to return result codes
270    // which are potentially greater than 1. This is not supported by
271    // the task framework.
272    if (countRejects.isPresent() && argParser.connectionArgumentsPresent())
273    {
274      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_COUNT_REJECTS_REQUIRES_OFFLINE.get(
275          countRejects.getLongIdentifier()));
276      return 1;
277    }
278
279    // Don't write non-error messages to console if quite
280    if (quietMode.isPresent()) {
281      out = new PrintStream(NullOutputStream.instance());
282    }
283
284    // Checks the version - if upgrade required, the tool is unusable
285    try
286    {
287      checkVersion();
288    }
289    catch (InitializationException e)
290    {
291      printWrappedText(err, e.getMessage());
292      return 1;
293    }
294
295    return process(argParser, initializeServer, out, err);
296  }
297
298  private void createArguments(LDAPConnectionArgumentParser argParser) throws ArgumentException
299  {
300      configClass =
301           new StringArgument("configclass", OPTION_SHORT_CONFIG_CLASS,
302                              OPTION_LONG_CONFIG_CLASS, true, false,
303                              true, INFO_CONFIGCLASS_PLACEHOLDER.get(),
304                              ConfigFileHandler.class.getName(), null,
305                              INFO_DESCRIPTION_CONFIG_CLASS.get());
306      configClass.setHidden(true);
307      argParser.addArgument(configClass);
308
309
310      configFile =
311           new StringArgument("configfile", 'f', "configFile", true, false,
312                              true, INFO_CONFIGFILE_PLACEHOLDER.get(), null,
313                              null,
314                              INFO_DESCRIPTION_CONFIG_FILE.get());
315      configFile.setHidden(true);
316      argParser.addArgument(configFile);
317
318
319      ldifFiles =
320           new StringArgument("ldiffile", OPTION_SHORT_LDIF_FILE,
321                              OPTION_LONG_LDIF_FILE, false, true, true,
322                              INFO_LDIFFILE_PLACEHOLDER.get(), null, null,
323                              INFO_LDIFIMPORT_DESCRIPTION_LDIF_FILE.get());
324      argParser.addArgument(ldifFiles);
325
326
327      templateFile =
328           new StringArgument("templatefile", 'A', "templateFile", false, false,
329                              true, INFO_TEMPLATE_FILE_PLACEHOLDER.get(), null,
330                              null,
331                              INFO_LDIFIMPORT_DESCRIPTION_TEMPLATE_FILE.get());
332      argParser.addArgument(templateFile);
333
334
335      append =
336           new BooleanArgument("append", 'a', "append",
337                               INFO_LDIFIMPORT_DESCRIPTION_APPEND.get());
338      argParser.addArgument(append);
339
340
341      replaceExisting =
342           new BooleanArgument(
343                   "replaceexisting", 'r', "replaceExisting",
344                   INFO_LDIFIMPORT_DESCRIPTION_REPLACE_EXISTING.get());
345      argParser.addArgument(replaceExisting);
346
347
348      backendID =
349           new StringArgument("backendid", 'n', "backendID", false, false, true,
350                              INFO_BACKENDNAME_PLACEHOLDER.get(), null, null,
351                              INFO_LDIFIMPORT_DESCRIPTION_BACKEND_ID.get());
352      argParser.addArgument(backendID);
353
354      clearBackend =
355          new BooleanArgument("clearbackend", 'F', "clearBackend",
356                              INFO_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND.get());
357      argParser.addArgument(clearBackend);
358
359
360      includeBranchStrings =
361           new StringArgument("includebranch", 'b', "includeBranch", false,
362                              true, true, INFO_BRANCH_DN_PLACEHOLDER.get(),
363                              null, null,
364                              INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_BRANCH.get());
365      argParser.addArgument(includeBranchStrings);
366
367
368      excludeBranchStrings =
369           new StringArgument("excludebranch", 'B', "excludeBranch", false,
370                              true, true, INFO_BRANCH_DN_PLACEHOLDER.get(),
371                              null, null,
372                              INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_BRANCH.get());
373      argParser.addArgument(excludeBranchStrings);
374
375
376      includeAttributeStrings =
377           new StringArgument(
378                   "includeattribute", 'i', "includeAttribute",
379                   false, true, true, INFO_ATTRIBUTE_PLACEHOLDER.get(), null,
380                   null,
381                   INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_ATTRIBUTE.get());
382      argParser.addArgument(includeAttributeStrings);
383
384
385      excludeAttributeStrings =
386           new StringArgument(
387                   "excludeattribute", 'e', "excludeAttribute",
388                   false, true, true, INFO_ATTRIBUTE_PLACEHOLDER.get(), null,
389                   null,
390                   INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_ATTRIBUTE.get());
391      argParser.addArgument(excludeAttributeStrings);
392
393
394      includeFilterStrings =
395           new StringArgument(
396                   "includefilter", 'I', "includeFilter",
397                   false, true, true, INFO_FILTER_PLACEHOLDER.get(), null, null,
398                   INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_FILTER.get());
399      argParser.addArgument(includeFilterStrings);
400
401
402      excludeFilterStrings =
403           new StringArgument("excludefilter", 'E', "excludeFilter",
404                              false, true, true, INFO_FILTER_PLACEHOLDER.get(),
405                              null, null,
406                              INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_FILTER.get());
407      argParser.addArgument(excludeFilterStrings);
408
409
410      rejectFile =
411           new StringArgument("rejectfile", 'R', "rejectFile", false, false,
412                              true, INFO_REJECT_FILE_PLACEHOLDER.get(), null,
413                              null,
414                              INFO_LDIFIMPORT_DESCRIPTION_REJECT_FILE.get());
415      argParser.addArgument(rejectFile);
416
417
418      skipFile =
419           new StringArgument("skipfile", null, "skipFile", false, false,
420                              true, INFO_SKIP_FILE_PLACEHOLDER.get(), null,
421                              null,
422                              INFO_LDIFIMPORT_DESCRIPTION_SKIP_FILE.get());
423      argParser.addArgument(skipFile);
424
425
426      overwrite =
427           new BooleanArgument("overwrite", 'O', "overwrite",
428                               INFO_LDIFIMPORT_DESCRIPTION_OVERWRITE.get());
429      argParser.addArgument(overwrite);
430
431
432      randomSeed =
433           new IntegerArgument("randomseed", OPTION_SHORT_RANDOM_SEED,
434                               OPTION_LONG_RANDOM_SEED, false, false,
435                               true, INFO_SEED_PLACEHOLDER.get(),
436                               0, null, false, 0, false, 0,
437                               INFO_LDIFIMPORT_DESCRIPTION_RANDOM_SEED.get());
438      argParser.addArgument(randomSeed);
439
440
441      skipSchemaValidation =
442           new BooleanArgument("skipschema", 'S', "skipSchemaValidation",
443                    INFO_LDIFIMPORT_DESCRIPTION_SKIP_SCHEMA_VALIDATION.get());
444      argParser.addArgument(skipSchemaValidation);
445
446
447      skipDNValidation =
448           new BooleanArgument("skipDNValidation", null, "skipDNValidation",
449                    INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION.get());
450      argParser.addArgument(skipDNValidation);
451
452
453      threadCount = new IntegerArgument("threadCount", null, "threadCount",
454              false, false, true,
455              INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER.get(),
456              0, null,
457              true, 1, true, Integer.MAX_VALUE,
458              INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT.get());
459      argParser.addArgument(threadCount);
460
461      tmpDirectory =
462           new StringArgument("tmpdirectory", null, "tmpdirectory", false,
463                   false, true, INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER.get(),
464                   "import-tmp",
465                    null, INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY.get());
466      argParser.addArgument(tmpDirectory);
467
468
469      countRejects =
470           new BooleanArgument("countrejects", null, "countRejects",
471                               INFO_LDIFIMPORT_DESCRIPTION_COUNT_REJECTS.get());
472      argParser.addArgument(countRejects);
473
474
475      isCompressed =
476           new BooleanArgument("iscompressed", 'c', "isCompressed",
477                               INFO_LDIFIMPORT_DESCRIPTION_IS_COMPRESSED.get());
478      argParser.addArgument(isCompressed);
479
480
481      isEncrypted =
482           new BooleanArgument("isencrypted", 'y', "isEncrypted",
483                               INFO_LDIFIMPORT_DESCRIPTION_IS_ENCRYPTED.get());
484      isEncrypted.setHidden(true); //See issue #27
485      argParser.addArgument(isEncrypted);
486
487
488      quietMode = new BooleanArgument("quietmode", OPTION_SHORT_QUIET,
489                                      OPTION_LONG_QUIET,
490                                      INFO_LDIFIMPORT_DESCRIPTION_QUIET.get());
491      argParser.addArgument(quietMode);
492
493
494      displayUsage = CommonArguments.getShowUsage();
495      argParser.addArgument(displayUsage);
496      argParser.setUsageArgument(displayUsage);
497  }
498
499  /** {@inheritDoc} */
500  @Override
501  public void addTaskAttributes(List<RawAttribute> attributes)
502  {
503    // Required attributes
504    addAttribute(attributes, ATTR_IMPORT_LDIF_FILE, ldifFiles.getValues());
505    addAttribute(attributes, ATTR_IMPORT_TEMPLATE_FILE, templateFile.getValue());
506    addAttribute(attributes, ATTR_IMPORT_RANDOM_SEED, randomSeed.getValue());
507    addAttribute(attributes, ATTR_IMPORT_THREAD_COUNT, threadCount.getValue());
508
509    // Optional attributes
510    addAttribute2(attributes, ATTR_IMPORT_APPEND, append);
511    addAttribute2(attributes, ATTR_IMPORT_REPLACE_EXISTING, replaceExisting);
512    addAttribute2(attributes, ATTR_IMPORT_BACKEND_ID, backendID);
513    addAttribute(attributes, ATTR_IMPORT_INCLUDE_ATTRIBUTE, includeAttributeStrings.getValues());
514    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_ATTRIBUTE, excludeAttributeStrings.getValues());
515    addAttribute(attributes, ATTR_IMPORT_INCLUDE_FILTER, includeFilterStrings.getValues());
516    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_FILTER, excludeFilterStrings.getValues());
517    addAttribute(attributes, ATTR_IMPORT_INCLUDE_BRANCH, includeBranchStrings.getValues());
518    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_BRANCH, excludeBranchStrings.getValues());
519    addAttribute2(attributes, ATTR_IMPORT_REJECT_FILE, rejectFile);
520    addAttribute2(attributes, ATTR_IMPORT_SKIP_FILE, skipFile);
521    addAttribute2(attributes, ATTR_IMPORT_OVERWRITE, overwrite);
522    addAttribute2(attributes, ATTR_IMPORT_SKIP_SCHEMA_VALIDATION, skipSchemaValidation);
523    addAttribute2(attributes, ATTR_IMPORT_TMP_DIRECTORY, tmpDirectory);
524    addAttribute2(attributes, ATTR_IMPORT_SKIP_DN_VALIDATION, skipDNValidation);
525    addAttribute2(attributes, ATTR_IMPORT_IS_COMPRESSED, isCompressed);
526    addAttribute2(attributes, ATTR_IMPORT_IS_ENCRYPTED, isEncrypted);
527    addAttribute2(attributes, ATTR_IMPORT_CLEAR_BACKEND, clearBackend);
528  }
529
530  private void addAttribute(List<RawAttribute> attributes, String attrName, String value)
531  {
532    if (value != null)
533    {
534      attributes.add(new LDAPAttribute(attrName, value));
535    }
536  }
537
538  private void addAttribute2(List<RawAttribute> attributes, String attrName, Argument arg)
539  {
540    final String value = arg.getValue();
541    if (value != null && !value.equals(arg.getDefaultValue()))
542    {
543      attributes.add(new LDAPAttribute(attrName, value));
544    }
545  }
546
547  private void addAttribute(List<RawAttribute> attributes, String attrName, List<String> attrValues)
548  {
549    if (attrValues != null && !attrValues.isEmpty())
550    {
551      attributes.add(new LDAPAttribute(attrName, attrValues));
552    }
553  }
554
555  /** {@inheritDoc} */
556  @Override
557  public String getTaskObjectclass() {
558    return "ds-task-import";
559  }
560
561  /** {@inheritDoc} */
562  @Override
563  public Class<?> getTaskClass() {
564    return ImportTask.class;
565  }
566
567  /** {@inheritDoc} */
568  @Override
569  protected int processLocal(boolean initializeServer,
570                           PrintStream out,
571                           PrintStream err) {
572
573
574    // Perform the initial bootstrap of the Directory Server and process the
575    // configuration.
576    DirectoryServer directoryServer = DirectoryServer.getInstance();
577    if (initializeServer)
578    {
579      try
580      {
581        DirectoryServer.bootstrapClient();
582        DirectoryServer.initializeJMX();
583      }
584      catch (Exception e)
585      {
586        printWrappedText(err, ERR_SERVER_BOOTSTRAP_ERROR.get(getExceptionMessage(e)));
587        return 1;
588      }
589
590      try
591      {
592        directoryServer.initializeConfiguration(configClass.getValue(),
593                                                configFile.getValue());
594      }
595      catch (InitializationException ie)
596      {
597        printWrappedText(err, ERR_CANNOT_LOAD_CONFIG.get(ie.getMessage()));
598        return 1;
599      }
600      catch (Exception e)
601      {
602        printWrappedText(err, ERR_CANNOT_LOAD_CONFIG.get(getExceptionMessage(e)));
603        return 1;
604      }
605
606
607
608      // Initialize the Directory Server schema elements.
609      try
610      {
611        directoryServer.initializeSchema();
612      }
613      catch (Exception e)
614      {
615        printWrappedText(err, ERR_CANNOT_LOAD_SCHEMA.get(getMessage(e)));
616        return 1;
617      }
618
619
620      // Initialize the Directory Server core configuration.
621      try
622      {
623        CoreConfigManager coreConfigManager = new CoreConfigManager(directoryServer.getServerContext());
624        coreConfigManager.initializeCoreConfig();
625      }
626      catch (Exception e)
627      {
628        printWrappedText(err, ERR_CANNOT_INITIALIZE_CORE_CONFIG.get(getMessage(e)));
629        return 1;
630      }
631
632
633      // Initialize the Directory Server crypto manager.
634      try
635      {
636        directoryServer.initializeCryptoManager();
637      }
638      catch (Exception e)
639      {
640        printWrappedText(err, ERR_CANNOT_INITIALIZE_CRYPTO_MANAGER.get(getMessage(e)));
641        return 1;
642      }
643
644
645      if (! quietMode.isPresent())
646      {
647        try
648        {
649          ErrorLogPublisher errorLogPublisher =
650              TextErrorLogPublisher.getToolStartupTextErrorPublisher(
651                  new TextWriter.STREAM(out));
652          ErrorLogger.getInstance().addLogPublisher(errorLogPublisher);
653        }
654        catch(Exception e)
655        {
656          err.println("Error installing the custom error logger: " +
657              stackTraceToSingleLineString(e));
658        }
659      }
660
661      // Initialize the root DNs.
662      try
663      {
664        directoryServer.initializeRootDNConfigManager();
665      }
666      catch (Exception e)
667      {
668        printWrappedText(err, ERR_CANNOT_INITIALIZE_ROOTDN_MANAGER.get(getMessage(e)));
669        return 1;
670      }
671
672      // Initialize the plugin manager.
673      try
674      {
675        HashSet<PluginType> pluginTypes = new HashSet<>(1);
676        directoryServer.initializePlugins(pluginTypes);
677      }
678      catch (Exception e)
679      {
680        printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PLUGINS.get(getMessage(e)));
681        return 1;
682      }
683
684      // Initialize the subentry manager.
685      try
686      {
687        directoryServer.initializeSubentryManager();
688      }
689      catch (InitializationException ie)
690      {
691        printWrappedText(err, ERR_CANNOT_INITIALIZE_SUBENTRY_MANAGER.get(ie.getMessage()));
692        return 1;
693      }
694
695      // Initialize all the password policy information.
696      try
697      {
698        directoryServer.initializeAuthenticationPolicyComponents();
699      }
700      catch (Exception e)
701      {
702        printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PWPOLICY.get(getMessage(e)));
703        return 1;
704      }
705    }
706
707    // Make sure that the plugin initialization is performed.
708    try
709    {
710      HashSet<PluginType> pluginTypes = new HashSet<>(1);
711      pluginTypes.add(PluginType.LDIF_IMPORT);
712      PluginConfigManager pluginConfigManager =
713              DirectoryServer.getPluginConfigManager();
714      pluginConfigManager.initializeUserPlugins(pluginTypes);
715    }
716    catch (Exception e)
717    {
718      printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PLUGINS.get(getMessage(e)));
719      return 1;
720    }
721
722    // See if there were any user-defined sets of include/exclude attributes or
723    // filters.  If so, then process them.
724    HashSet<AttributeType> excludeAttributes;
725    boolean excludeAllUserAttributes = false;
726    boolean excludeAllOperationalAttributes = false;
727    if (excludeAttributeStrings == null)
728    {
729      excludeAttributes = null;
730    }
731    else
732    {
733      excludeAttributes = new HashSet<>();
734      for (String attrName : excludeAttributeStrings.getValues())
735      {
736        String        lowerName = attrName.toLowerCase();
737        if(lowerName.equals("*"))
738        {
739          excludeAllUserAttributes = true;
740        }
741        else if(lowerName.equals("+"))
742        {
743          excludeAllOperationalAttributes = true;
744        }
745        else
746        {
747          excludeAttributes.add(DirectoryServer.getAttributeTypeOrDefault(lowerName, attrName));
748        }
749      }
750    }
751
752    HashSet<AttributeType> includeAttributes;
753    boolean includeAllUserAttributes = false;
754    boolean includeAllOperationalAttributes = false;
755    if (includeAttributeStrings == null)
756    {
757      includeAttributes = null;
758    }
759    else
760    {
761      includeAttributes = new HashSet<>();
762      for (String attrName : includeAttributeStrings.getValues())
763      {
764        String        lowerName = attrName.toLowerCase();
765         if(lowerName.equals("*"))
766        {
767          includeAllUserAttributes = true;
768        }
769        else if(lowerName.equals("+"))
770        {
771          includeAllOperationalAttributes = true;
772        }
773        else
774        {
775          includeAttributes.add(DirectoryServer.getAttributeTypeOrDefault(lowerName, attrName));
776        }
777      }
778    }
779
780    ArrayList<SearchFilter> excludeFilters;
781    if (excludeFilterStrings == null)
782    {
783      excludeFilters = null;
784    }
785    else
786    {
787      excludeFilters = new ArrayList<>();
788      for (String filterString : excludeFilterStrings.getValues())
789      {
790        try
791        {
792          excludeFilters.add(SearchFilter.createFilterFromString(filterString));
793        }
794        catch (DirectoryException de)
795        {
796          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, de.getMessageObject());
797          return 1;
798        }
799        catch (Exception e)
800        {
801          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, getExceptionMessage(e));
802          return 1;
803        }
804      }
805    }
806
807    ArrayList<SearchFilter> includeFilters;
808    if (includeFilterStrings == null)
809    {
810      includeFilters = null;
811    }
812    else
813    {
814      includeFilters = new ArrayList<>();
815      for (String filterString : includeFilterStrings.getValues())
816      {
817        try
818        {
819          includeFilters.add(SearchFilter.createFilterFromString(filterString));
820        }
821        catch (DirectoryException de)
822        {
823          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, de.getMessageObject());
824          return 1;
825        }
826        catch (Exception e)
827        {
828          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, getExceptionMessage(e));
829          return 1;
830        }
831      }
832    }
833
834
835    // Get information about the backends defined in the server.  Iterate
836    // through them, finding the one backend into which the LDIF should be
837    // imported and finding backends with subordinate base DNs that should be
838    // excluded from the import.
839    Backend<?> backend = null;
840    Set<DN> defaultIncludeBranches = null;
841    Set<DN> excludeBranches = new HashSet<>();
842    Set<DN> includeBranches = new HashSet<>();
843
844    if (includeBranchStrings.isPresent())
845    {
846      for (String s : includeBranchStrings.getValues())
847      {
848        DN includeBranch;
849        try
850        {
851          includeBranch = DN.valueOf(s);
852        }
853        catch (DirectoryException de)
854        {
855          logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, de.getMessageObject());
856          return 1;
857        }
858        catch (Exception e)
859        {
860          logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, getExceptionMessage(e));
861          return 1;
862        }
863
864        includeBranches.add(includeBranch);
865      }
866    }
867
868    ArrayList<Backend>     backendList = new ArrayList<>();
869    ArrayList<BackendCfg>  entryList   = new ArrayList<>();
870    ArrayList<List<DN>> dnList = new ArrayList<>();
871    int code = BackendToolUtils.getBackends(backendList, entryList, dnList);
872    if (code != 0)
873    {
874      return code;
875    }
876
877    int numBackends = backendList.size();
878    for (int i=0; i < numBackends; i++)
879    {
880      Backend<?> b = backendList.get(i);
881
882      if(backendID.isPresent())
883      {
884        if (! backendID.getValue().equals(b.getBackendID()))
885        {
886          continue;
887        }
888      }
889      else
890      {
891        if (!useBackend(includeBranches, dnList.get(i)))
892        {
893          continue;
894        }
895      }
896
897      if (backend == null)
898      {
899        backend                = b;
900        defaultIncludeBranches = new HashSet<>(dnList.get(i));
901      }
902      else
903      {
904        logger.error(ERR_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID);
905        return 1;
906      }
907    }
908
909    if (backend == null)
910    {
911      logger.error(ERR_LDIFIMPORT_NO_BACKENDS_FOR_ID);
912      return 1;
913    }
914    else if (!backend.supports(BackendOperation.LDIF_IMPORT))
915    {
916      logger.error(ERR_LDIFIMPORT_CANNOT_IMPORT, backendID.getValue());
917      return 1;
918    }
919
920    for (List<DN> baseList : dnList)
921    {
922      for (DN baseDN : baseList)
923      {
924        for (DN importBase : defaultIncludeBranches)
925        {
926          if (!baseDN.equals(importBase) && baseDN.isDescendantOf(importBase))
927          {
928            excludeBranches.add(baseDN);
929            break;
930          }
931        }
932      }
933    }
934
935    // Make sure that if the "backendID" argument was provided, no include base
936    // was included, the
937    // "clearBackend" argument was also provided if there are more then one
938    // baseDNs for the backend being imported.
939
940    if(backendID.isPresent() && !includeBranchStrings.isPresent() &&
941       !append.isPresent() &&
942        defaultIncludeBranches.size() > 1 &&
943        !clearBackend.isPresent())
944    {
945      StringBuilder builder = join(backend.getBaseDNs(), " / ");
946      printWrappedText(err, ERR_LDIFIMPORT_MISSING_CLEAR_BACKEND.get(builder, clearBackend.getLongIdentifier()));
947      return 1;
948    }
949
950    for (String s : excludeBranchStrings.getValues())
951    {
952      DN excludeBranch;
953      try
954      {
955        excludeBranch = DN.valueOf(s);
956      }
957      catch (DirectoryException de)
958      {
959        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, de.getMessageObject());
960        return 1;
961      }
962      catch (Exception e)
963      {
964        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, getExceptionMessage(e));
965        return 1;
966      }
967
968      excludeBranches.add(excludeBranch);
969    }
970
971    if (! includeBranchStrings.isPresent())
972    {
973      includeBranches = defaultIncludeBranches;
974    }
975    else
976    {
977      // Make sure the selected backend will handle all the include branches
978      for(DN includeBranch : includeBranches)
979      {
980        if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches,
981                                   excludeBranches))
982        {
983          logger.error(ERR_LDIFIMPORT_INVALID_INCLUDE_BASE, includeBranch, backendID.getValue());
984          return 1;
985        }
986      }
987    }
988
989
990    // See if the data should be read from LDIF files or generated via MakeLDIF.
991    LDIFImportConfig importConfig;
992    if (ldifFiles.isPresent())
993    {
994      ArrayList<String> fileList = new ArrayList<>(ldifFiles.getValues());
995      int badFileCount = 0;
996      for (String pathname : fileList)
997      {
998        File f = new File(pathname);
999        if (!f.canRead())
1000        {
1001          logger.error(ERR_LDIFIMPORT_CANNOT_READ_FILE, pathname);
1002          badFileCount++;
1003        }
1004      }
1005      if (badFileCount > 0)
1006      {
1007        return 1;
1008      }
1009      importConfig = new LDIFImportConfig(fileList);
1010    }
1011    else
1012    {
1013      Random random = newRandom();
1014
1015      String resourcePath = DirectoryServer.getInstanceRoot() + File.separator +
1016                            PATH_MAKELDIF_RESOURCE_DIR;
1017      TemplateFile tf = new TemplateFile(resourcePath, random);
1018
1019      ArrayList<LocalizableMessage> warnings = new ArrayList<>();
1020      try
1021      {
1022        tf.parse(templateFile.getValue(), warnings);
1023      }
1024      catch (Exception e)
1025      {
1026        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_TEMPLATE_FILE, templateFile.getValue(), e.getMessage());
1027        return 1;
1028      }
1029
1030      importConfig = new LDIFImportConfig(tf);
1031    }
1032
1033
1034      // Create the LDIF import configuration to use when reading the LDIF.
1035      importConfig.setAppendToExistingData(append.isPresent());
1036      importConfig.setReplaceExistingEntries(replaceExisting.isPresent());
1037      importConfig.setCompressed(isCompressed.isPresent());
1038      importConfig.setClearBackend(clearBackend.isPresent());
1039      importConfig.setEncrypted(isEncrypted.isPresent());
1040      importConfig.setExcludeAttributes(excludeAttributes);
1041      importConfig.setExcludeBranches(excludeBranches);
1042      importConfig.setExcludeFilters(excludeFilters);
1043      importConfig.setIncludeAttributes(includeAttributes);
1044      importConfig.setIncludeBranches(includeBranches);
1045      importConfig.setIncludeFilters(includeFilters);
1046      importConfig.setValidateSchema(!skipSchemaValidation.isPresent());
1047      importConfig.setSkipDNValidation(skipDNValidation.isPresent());
1048      importConfig.setTmpDirectory(tmpDirectory.getValue());
1049
1050      try
1051      {
1052          importConfig.setThreadCount(threadCount.getIntValue());
1053      }
1054      catch(Exception e)
1055      {
1056          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT,
1057                  threadCount.getValue(), e.getMessage());
1058          return 1;
1059      }
1060
1061    importConfig.setBufferSize(LDIF_BUFFER_SIZE);
1062    importConfig.setExcludeAllUserAttributes(excludeAllUserAttributes);
1063    importConfig.setExcludeAllOperationalAttributes(excludeAllOperationalAttributes);
1064    importConfig.setIncludeAllOpAttributes(includeAllOperationalAttributes);
1065    importConfig.setIncludeAllUserAttributes(includeAllUserAttributes);
1066
1067    // FIXME -- Should this be conditional?
1068    importConfig.setInvokeImportPlugins(true);
1069
1070    if (rejectFile != null)
1071    {
1072      try
1073      {
1074        ExistingFileBehavior existingBehavior = overwrite.isPresent()
1075            ? ExistingFileBehavior.OVERWRITE
1076            : ExistingFileBehavior.APPEND;
1077
1078        importConfig.writeRejectedEntries(rejectFile.getValue(),
1079                                          existingBehavior);
1080      }
1081      catch (Exception e)
1082      {
1083        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_REJECTS_FILE, rejectFile.getValue(), getExceptionMessage(e));
1084        return 1;
1085      }
1086    }
1087
1088    if (skipFile != null)
1089    {
1090      try
1091      {
1092        ExistingFileBehavior existingBehavior = overwrite.isPresent()
1093            ? ExistingFileBehavior.OVERWRITE
1094            : ExistingFileBehavior.APPEND;
1095
1096        importConfig.writeSkippedEntries(skipFile.getValue(),
1097                                          existingBehavior);
1098      }
1099      catch (Exception e)
1100      {
1101        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_SKIP_FILE, skipFile.getValue(), getExceptionMessage(e));
1102        return 1;
1103      }
1104    }
1105
1106    // Get the set of base DNs for the backend as an array.
1107    DN[] baseDNs = new DN[defaultIncludeBranches.size()];
1108    defaultIncludeBranches.toArray(baseDNs);
1109
1110
1111    // Acquire an exclusive lock for the backend.
1112    try
1113    {
1114      String lockFile = LockFileManager.getBackendLockFileName(backend);
1115      StringBuilder failureReason = new StringBuilder();
1116      if (! LockFileManager.acquireExclusiveLock(lockFile, failureReason))
1117      {
1118        logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), failureReason);
1119        return 1;
1120      }
1121    }
1122    catch (Exception e)
1123    {
1124      logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
1125      return 1;
1126    }
1127
1128
1129    // Launch the import.
1130    int retCode = 0;
1131    try
1132    {
1133      LDIFImportResult importResult =
1134          backend.importLDIF(importConfig, DirectoryServer.getInstance().getServerContext());
1135      if (countRejects.isPresent())
1136      {
1137        if (importResult.getEntriesRejected() > Integer.MAX_VALUE)
1138        {
1139          retCode = Integer.MAX_VALUE;
1140        }
1141        else
1142        {
1143          retCode = (int) importResult.getEntriesRejected();
1144        }
1145      }
1146    }
1147    catch (DirectoryException de)
1148    {
1149      LocalizableMessage msg;
1150      if (de.getResultCode() == ResultCode.CONSTRAINT_VIOLATION)
1151      {
1152        msg = ERR_LDIFIMPORT_ERROR_CONSTRAINT_VIOLATION.get();
1153      }
1154      else
1155      {
1156        msg = de.getMessageObject();
1157      }
1158      logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg));
1159      retCode = 1;
1160    }
1161    catch (Exception e)
1162    {
1163      logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT, getExceptionMessage(e));
1164      retCode = 1;
1165    }
1166
1167
1168    // Release the exclusive lock on the backend.
1169    try
1170    {
1171      String lockFile = LockFileManager.getBackendLockFileName(backend);
1172      StringBuilder failureReason = new StringBuilder();
1173      if (! LockFileManager.releaseLock(lockFile, failureReason))
1174      {
1175        logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), failureReason);
1176        retCode = 1;
1177      }
1178    }
1179    catch (Exception e)
1180    {
1181      logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
1182      retCode = 1;
1183    }
1184
1185
1186    // Clean up after the import by closing the import config.
1187    importConfig.close();
1188    return retCode;
1189  }
1190
1191  private Object getMessage(Exception e)
1192  {
1193    try
1194    {
1195      throw e;
1196    }
1197    catch (ConfigException | InitializationException e2)
1198    {
1199      return e2.getMessage();
1200    }
1201    catch (Exception e2)
1202    {
1203      return getExceptionMessage(e2);
1204    }
1205  }
1206
1207  private boolean useBackend(Set<DN> includeBranches, List<DN> dnlist)
1208  {
1209    for (DN baseDN : dnlist)
1210    {
1211      for (DN includeDN : includeBranches)
1212      {
1213        if (baseDN.isAncestorOf(includeDN))
1214        {
1215          return true;
1216        }
1217      }
1218    }
1219    return false;
1220  }
1221
1222  private StringBuilder join(final DN[] baseDNs, final String separator)
1223  {
1224    final StringBuilder builder = new StringBuilder();
1225    builder.append(baseDNs[0]);
1226    for (int i = 1; i < baseDNs.length; i++)
1227    {
1228      builder.append(separator);
1229      builder.append(baseDNs[i]);
1230    }
1231    return builder;
1232  }
1233
1234  private Random newRandom()
1235  {
1236    if (randomSeed.isPresent())
1237    {
1238      try
1239      {
1240        return new Random(randomSeed.getIntValue());
1241      }
1242      catch (Exception ignored)
1243      {
1244        // ignore
1245      }
1246    }
1247    return new Random();
1248  }
1249
1250  /** {@inheritDoc} */
1251  @Override
1252  public String getTaskId() {
1253    // NYI.
1254    return null;
1255  }
1256}