001/* 002 * CDDL HEADER START 003 * 004 * The contents of this file are subject to the terms of the 005 * Common Development and Distribution License, Version 1.0 only 006 * (the "License"). You may not use this file except in compliance 007 * with the License. 008 * 009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt 010 * or http://forgerock.org/license/CDDLv1.0.html. 011 * See the License for the specific language governing permissions 012 * and limitations under the License. 013 * 014 * When distributing Covered Code, include this CDDL HEADER in each 015 * file and include the License file at legal-notices/CDDLv1_0.txt. 016 * If applicable, add the following below this CDDL HEADER, with the 017 * fields enclosed by brackets "[]" replaced with your own identifying 018 * information: 019 * Portions Copyright [yyyy] [name of copyright owner] 020 * 021 * CDDL HEADER END 022 * 023 * 024 * Copyright 2006-2010 Sun Microsystems, Inc. 025 * Portions Copyright 2011-2015 ForgeRock AS. 026 */ 027package org.opends.server.tools; 028 029import static org.opends.messages.ToolMessages.*; 030import static org.opends.server.config.ConfigConstants.*; 031import static org.opends.server.util.StaticUtils.*; 032 033import static com.forgerock.opendj.cli.ArgumentConstants.*; 034import static com.forgerock.opendj.cli.Utils.*; 035 036import java.io.File; 037import java.io.OutputStream; 038import java.io.PrintStream; 039import java.util.ArrayList; 040import java.util.HashSet; 041import java.util.List; 042import java.util.Random; 043import java.util.Set; 044 045import org.forgerock.i18n.LocalizableMessage; 046import org.forgerock.i18n.slf4j.LocalizedLogger; 047import org.forgerock.opendj.config.server.ConfigException; 048import org.forgerock.opendj.ldap.ResultCode; 049import org.opends.server.admin.std.server.BackendCfg; 050import org.opends.server.api.Backend; 051import org.opends.server.api.Backend.BackendOperation; 052import org.opends.server.api.plugin.PluginType; 053import org.opends.server.core.CoreConfigManager; 054import org.opends.server.core.DirectoryServer; 055import org.opends.server.core.LockFileManager; 056import org.opends.server.core.PluginConfigManager; 057import org.opends.server.extensions.ConfigFileHandler; 058import org.opends.server.loggers.ErrorLogPublisher; 059import org.opends.server.loggers.ErrorLogger; 060import org.opends.server.loggers.JDKLogging; 061import org.opends.server.loggers.TextErrorLogPublisher; 062import org.opends.server.loggers.TextWriter; 063import org.opends.server.protocols.ldap.LDAPAttribute; 064import org.opends.server.tasks.ImportTask; 065import org.opends.server.tools.makeldif.TemplateFile; 066import org.opends.server.tools.tasks.TaskTool; 067import org.opends.server.types.AttributeType; 068import org.opends.server.types.DN; 069import org.opends.server.types.DirectoryException; 070import org.opends.server.types.ExistingFileBehavior; 071import org.opends.server.types.InitializationException; 072import org.opends.server.types.LDIFImportConfig; 073import org.opends.server.types.LDIFImportResult; 074import org.opends.server.types.NullOutputStream; 075import org.opends.server.types.RawAttribute; 076import org.opends.server.types.SearchFilter; 077import org.opends.server.util.args.LDAPConnectionArgumentParser; 078 079import com.forgerock.opendj.cli.Argument; 080import com.forgerock.opendj.cli.ArgumentException; 081import com.forgerock.opendj.cli.BooleanArgument; 082import com.forgerock.opendj.cli.ClientException; 083import com.forgerock.opendj.cli.CommonArguments; 084import com.forgerock.opendj.cli.IntegerArgument; 085import com.forgerock.opendj.cli.StringArgument; 086 087/** 088 * This program provides a utility that may be used to import the contents of an 089 * LDIF file into a Directory Server backend. This will be a process that is 090 * intended to run separate from Directory Server and not internally within the 091 * server process (e.g., via the tasks interface). 092 */ 093public class ImportLDIF extends TaskTool { 094 095 private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); 096 097 /** 098 * The buffer size that should be used when reading data from LDIF. 099 */ 100 public static final int LDIF_BUFFER_SIZE = 1048576; 101 102 103 /** 104 * The main method for ImportLDIF tool. 105 * 106 * @param args The command-line arguments provided to this program. 107 */ 108 public static void main(String[] args) 109 { 110 int retCode = mainImportLDIF(args, true, System.out, System.err); 111 112 if(retCode != 0) 113 { 114 System.exit(filterExitCode(retCode)); 115 } 116 } 117 118 /** 119 * Processes the command-line arguments and invokes the import process. 120 * 121 * @param args The command-line arguments provided to this program. 122 * @return The error code. 123 */ 124 public static int mainImportLDIF(String[] args) 125 { 126 return mainImportLDIF(args, true, System.out, System.err); 127 } 128 129 /** 130 * Processes the command-line arguments and invokes the import process. 131 * 132 * @param args The command-line arguments provided to this 133 * program. 134 * @param initializeServer Indicates whether to initialize the server. 135 * @param outStream The output stream to use for standard output, or 136 * {@code null} if standard output is not needed. 137 * @param errStream The output stream to use for standard error, or 138 * {@code null} if standard error is not needed. 139 * 140 * @return The error code. 141 */ 142 public static int mainImportLDIF(String[] args, boolean initializeServer, 143 OutputStream outStream, 144 OutputStream errStream) 145 { 146 ImportLDIF tool = new ImportLDIF(); 147 return tool.process(args, initializeServer, outStream, errStream); 148 } 149 150 /** Define the command-line arguments that may be used with this program. */ 151 private BooleanArgument append; 152 private BooleanArgument countRejects; 153 private BooleanArgument displayUsage; 154 private BooleanArgument isCompressed; 155 private BooleanArgument isEncrypted; 156 private BooleanArgument overwrite; 157 private BooleanArgument quietMode; 158 private BooleanArgument replaceExisting; 159 private BooleanArgument skipSchemaValidation; 160 private BooleanArgument clearBackend; 161 private IntegerArgument randomSeed; 162 private StringArgument backendID; 163 private StringArgument configClass; 164 private StringArgument configFile; 165 private StringArgument excludeAttributeStrings; 166 private StringArgument excludeBranchStrings; 167 private StringArgument excludeFilterStrings; 168 private StringArgument includeAttributeStrings; 169 private StringArgument includeBranchStrings; 170 private StringArgument includeFilterStrings; 171 private StringArgument ldifFiles; 172 private StringArgument rejectFile; 173 private StringArgument skipFile; 174 private StringArgument templateFile; 175 private BooleanArgument skipDNValidation; 176 private IntegerArgument threadCount; 177 private StringArgument tmpDirectory; 178 179 private int process(String[] args, boolean initializeServer, 180 OutputStream outStream, OutputStream errStream) { 181 182 PrintStream out = NullOutputStream.wrapOrNullStream(outStream); 183 PrintStream err = NullOutputStream.wrapOrNullStream(errStream); 184 JDKLogging.disableLogging(); 185 186 // FIXME -- Need to add a mechanism for verifying the file signature. 187 188 189 // Create the command-line argument parser for use with this program. 190 LDAPConnectionArgumentParser argParser = 191 createArgParser("org.opends.server.tools.ImportLDIF", 192 INFO_LDIFIMPORT_TOOL_DESCRIPTION.get()); 193 argParser.setShortToolDescription(REF_SHORT_DESC_IMPORT_LDIF.get()); 194 195 // Initialize all the command-line argument types and register them with the 196 // parser. 197 try 198 { 199 createArguments(argParser); 200 } 201 catch (ArgumentException ae) 202 { 203 printWrappedText(err, ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage())); 204 return 1; 205 } 206 207 // Init the default values so that they can appear also on the usage. 208 try 209 { 210 argParser.getArguments().initArgumentsWithConfiguration(); 211 } 212 catch (ConfigException ce) 213 { 214 // Ignore. 215 } 216 217 // Parse the command-line arguments provided to this program. 218 try 219 { 220 argParser.parseArguments(args); 221 validateTaskArgs(); 222 } 223 catch (ArgumentException ae) 224 { 225 argParser.displayMessageAndUsageReference(err, ERR_ERROR_PARSING_ARGS.get(ae.getMessage())); 226 return 1; 227 } 228 catch (ClientException ce) 229 { 230 // No need to display the usage since the problem comes with a provided value. 231 printWrappedText(err, ce.getMessageObject()); 232 return 1; 233 } 234 235 236 if (argParser.usageOrVersionDisplayed()) 237 { 238 return 0; 239 } 240 241 242 // Make sure that either the "ldifFile" argument or the "templateFile" 243 // argument was provided, but not both. 244 if (ldifFiles.isPresent()) 245 { 246 if (templateFile.isPresent()) 247 { 248 printWrappedText(err, 249 ERR_LDIFIMPORT_CONFLICTING_OPTIONS.get(ldifFiles.getLongIdentifier(), templateFile.getLongIdentifier())); 250 return 1; 251 } 252 } 253 else if (! templateFile.isPresent()) 254 { 255 argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_REQUIRED_ARGUMENT.get( 256 ldifFiles.getLongIdentifier(), templateFile.getLongIdentifier())); 257 return 1; 258 } 259 260 // Make sure that either the "includeBranchStrings" argument or the 261 // "backendID" argument was provided. 262 if(!includeBranchStrings.isPresent() && !backendID.isPresent()) 263 { 264 argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_BACKEND_ARGUMENT.get( 265 includeBranchStrings.getLongIdentifier(), backendID.getLongIdentifier())); 266 return 1; 267 } 268 269 // Count rejects option requires the ability to return result codes 270 // which are potentially greater than 1. This is not supported by 271 // the task framework. 272 if (countRejects.isPresent() && argParser.connectionArgumentsPresent()) 273 { 274 argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_COUNT_REJECTS_REQUIRES_OFFLINE.get( 275 countRejects.getLongIdentifier())); 276 return 1; 277 } 278 279 // Don't write non-error messages to console if quite 280 if (quietMode.isPresent()) { 281 out = new PrintStream(NullOutputStream.instance()); 282 } 283 284 // Checks the version - if upgrade required, the tool is unusable 285 try 286 { 287 checkVersion(); 288 } 289 catch (InitializationException e) 290 { 291 printWrappedText(err, e.getMessage()); 292 return 1; 293 } 294 295 return process(argParser, initializeServer, out, err); 296 } 297 298 private void createArguments(LDAPConnectionArgumentParser argParser) throws ArgumentException 299 { 300 configClass = 301 new StringArgument("configclass", OPTION_SHORT_CONFIG_CLASS, 302 OPTION_LONG_CONFIG_CLASS, true, false, 303 true, INFO_CONFIGCLASS_PLACEHOLDER.get(), 304 ConfigFileHandler.class.getName(), null, 305 INFO_DESCRIPTION_CONFIG_CLASS.get()); 306 configClass.setHidden(true); 307 argParser.addArgument(configClass); 308 309 310 configFile = 311 new StringArgument("configfile", 'f', "configFile", true, false, 312 true, INFO_CONFIGFILE_PLACEHOLDER.get(), null, 313 null, 314 INFO_DESCRIPTION_CONFIG_FILE.get()); 315 configFile.setHidden(true); 316 argParser.addArgument(configFile); 317 318 319 ldifFiles = 320 new StringArgument("ldiffile", OPTION_SHORT_LDIF_FILE, 321 OPTION_LONG_LDIF_FILE, false, true, true, 322 INFO_LDIFFILE_PLACEHOLDER.get(), null, null, 323 INFO_LDIFIMPORT_DESCRIPTION_LDIF_FILE.get()); 324 argParser.addArgument(ldifFiles); 325 326 327 templateFile = 328 new StringArgument("templatefile", 'A', "templateFile", false, false, 329 true, INFO_TEMPLATE_FILE_PLACEHOLDER.get(), null, 330 null, 331 INFO_LDIFIMPORT_DESCRIPTION_TEMPLATE_FILE.get()); 332 argParser.addArgument(templateFile); 333 334 335 append = 336 new BooleanArgument("append", 'a', "append", 337 INFO_LDIFIMPORT_DESCRIPTION_APPEND.get()); 338 argParser.addArgument(append); 339 340 341 replaceExisting = 342 new BooleanArgument( 343 "replaceexisting", 'r', "replaceExisting", 344 INFO_LDIFIMPORT_DESCRIPTION_REPLACE_EXISTING.get()); 345 argParser.addArgument(replaceExisting); 346 347 348 backendID = 349 new StringArgument("backendid", 'n', "backendID", false, false, true, 350 INFO_BACKENDNAME_PLACEHOLDER.get(), null, null, 351 INFO_LDIFIMPORT_DESCRIPTION_BACKEND_ID.get()); 352 argParser.addArgument(backendID); 353 354 clearBackend = 355 new BooleanArgument("clearbackend", 'F', "clearBackend", 356 INFO_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND.get()); 357 argParser.addArgument(clearBackend); 358 359 360 includeBranchStrings = 361 new StringArgument("includebranch", 'b', "includeBranch", false, 362 true, true, INFO_BRANCH_DN_PLACEHOLDER.get(), 363 null, null, 364 INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_BRANCH.get()); 365 argParser.addArgument(includeBranchStrings); 366 367 368 excludeBranchStrings = 369 new StringArgument("excludebranch", 'B', "excludeBranch", false, 370 true, true, INFO_BRANCH_DN_PLACEHOLDER.get(), 371 null, null, 372 INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_BRANCH.get()); 373 argParser.addArgument(excludeBranchStrings); 374 375 376 includeAttributeStrings = 377 new StringArgument( 378 "includeattribute", 'i', "includeAttribute", 379 false, true, true, INFO_ATTRIBUTE_PLACEHOLDER.get(), null, 380 null, 381 INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_ATTRIBUTE.get()); 382 argParser.addArgument(includeAttributeStrings); 383 384 385 excludeAttributeStrings = 386 new StringArgument( 387 "excludeattribute", 'e', "excludeAttribute", 388 false, true, true, INFO_ATTRIBUTE_PLACEHOLDER.get(), null, 389 null, 390 INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_ATTRIBUTE.get()); 391 argParser.addArgument(excludeAttributeStrings); 392 393 394 includeFilterStrings = 395 new StringArgument( 396 "includefilter", 'I', "includeFilter", 397 false, true, true, INFO_FILTER_PLACEHOLDER.get(), null, null, 398 INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_FILTER.get()); 399 argParser.addArgument(includeFilterStrings); 400 401 402 excludeFilterStrings = 403 new StringArgument("excludefilter", 'E', "excludeFilter", 404 false, true, true, INFO_FILTER_PLACEHOLDER.get(), 405 null, null, 406 INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_FILTER.get()); 407 argParser.addArgument(excludeFilterStrings); 408 409 410 rejectFile = 411 new StringArgument("rejectfile", 'R', "rejectFile", false, false, 412 true, INFO_REJECT_FILE_PLACEHOLDER.get(), null, 413 null, 414 INFO_LDIFIMPORT_DESCRIPTION_REJECT_FILE.get()); 415 argParser.addArgument(rejectFile); 416 417 418 skipFile = 419 new StringArgument("skipfile", null, "skipFile", false, false, 420 true, INFO_SKIP_FILE_PLACEHOLDER.get(), null, 421 null, 422 INFO_LDIFIMPORT_DESCRIPTION_SKIP_FILE.get()); 423 argParser.addArgument(skipFile); 424 425 426 overwrite = 427 new BooleanArgument("overwrite", 'O', "overwrite", 428 INFO_LDIFIMPORT_DESCRIPTION_OVERWRITE.get()); 429 argParser.addArgument(overwrite); 430 431 432 randomSeed = 433 new IntegerArgument("randomseed", OPTION_SHORT_RANDOM_SEED, 434 OPTION_LONG_RANDOM_SEED, false, false, 435 true, INFO_SEED_PLACEHOLDER.get(), 436 0, null, false, 0, false, 0, 437 INFO_LDIFIMPORT_DESCRIPTION_RANDOM_SEED.get()); 438 argParser.addArgument(randomSeed); 439 440 441 skipSchemaValidation = 442 new BooleanArgument("skipschema", 'S', "skipSchemaValidation", 443 INFO_LDIFIMPORT_DESCRIPTION_SKIP_SCHEMA_VALIDATION.get()); 444 argParser.addArgument(skipSchemaValidation); 445 446 447 skipDNValidation = 448 new BooleanArgument("skipDNValidation", null, "skipDNValidation", 449 INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION.get()); 450 argParser.addArgument(skipDNValidation); 451 452 453 threadCount = new IntegerArgument("threadCount", null, "threadCount", 454 false, false, true, 455 INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER.get(), 456 0, null, 457 true, 1, true, Integer.MAX_VALUE, 458 INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT.get()); 459 argParser.addArgument(threadCount); 460 461 tmpDirectory = 462 new StringArgument("tmpdirectory", null, "tmpdirectory", false, 463 false, true, INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER.get(), 464 "import-tmp", 465 null, INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY.get()); 466 argParser.addArgument(tmpDirectory); 467 468 469 countRejects = 470 new BooleanArgument("countrejects", null, "countRejects", 471 INFO_LDIFIMPORT_DESCRIPTION_COUNT_REJECTS.get()); 472 argParser.addArgument(countRejects); 473 474 475 isCompressed = 476 new BooleanArgument("iscompressed", 'c', "isCompressed", 477 INFO_LDIFIMPORT_DESCRIPTION_IS_COMPRESSED.get()); 478 argParser.addArgument(isCompressed); 479 480 481 isEncrypted = 482 new BooleanArgument("isencrypted", 'y', "isEncrypted", 483 INFO_LDIFIMPORT_DESCRIPTION_IS_ENCRYPTED.get()); 484 isEncrypted.setHidden(true); //See issue #27 485 argParser.addArgument(isEncrypted); 486 487 488 quietMode = new BooleanArgument("quietmode", OPTION_SHORT_QUIET, 489 OPTION_LONG_QUIET, 490 INFO_LDIFIMPORT_DESCRIPTION_QUIET.get()); 491 argParser.addArgument(quietMode); 492 493 494 displayUsage = CommonArguments.getShowUsage(); 495 argParser.addArgument(displayUsage); 496 argParser.setUsageArgument(displayUsage); 497 } 498 499 /** {@inheritDoc} */ 500 @Override 501 public void addTaskAttributes(List<RawAttribute> attributes) 502 { 503 // Required attributes 504 addAttribute(attributes, ATTR_IMPORT_LDIF_FILE, ldifFiles.getValues()); 505 addAttribute(attributes, ATTR_IMPORT_TEMPLATE_FILE, templateFile.getValue()); 506 addAttribute(attributes, ATTR_IMPORT_RANDOM_SEED, randomSeed.getValue()); 507 addAttribute(attributes, ATTR_IMPORT_THREAD_COUNT, threadCount.getValue()); 508 509 // Optional attributes 510 addAttribute2(attributes, ATTR_IMPORT_APPEND, append); 511 addAttribute2(attributes, ATTR_IMPORT_REPLACE_EXISTING, replaceExisting); 512 addAttribute2(attributes, ATTR_IMPORT_BACKEND_ID, backendID); 513 addAttribute(attributes, ATTR_IMPORT_INCLUDE_ATTRIBUTE, includeAttributeStrings.getValues()); 514 addAttribute(attributes, ATTR_IMPORT_EXCLUDE_ATTRIBUTE, excludeAttributeStrings.getValues()); 515 addAttribute(attributes, ATTR_IMPORT_INCLUDE_FILTER, includeFilterStrings.getValues()); 516 addAttribute(attributes, ATTR_IMPORT_EXCLUDE_FILTER, excludeFilterStrings.getValues()); 517 addAttribute(attributes, ATTR_IMPORT_INCLUDE_BRANCH, includeBranchStrings.getValues()); 518 addAttribute(attributes, ATTR_IMPORT_EXCLUDE_BRANCH, excludeBranchStrings.getValues()); 519 addAttribute2(attributes, ATTR_IMPORT_REJECT_FILE, rejectFile); 520 addAttribute2(attributes, ATTR_IMPORT_SKIP_FILE, skipFile); 521 addAttribute2(attributes, ATTR_IMPORT_OVERWRITE, overwrite); 522 addAttribute2(attributes, ATTR_IMPORT_SKIP_SCHEMA_VALIDATION, skipSchemaValidation); 523 addAttribute2(attributes, ATTR_IMPORT_TMP_DIRECTORY, tmpDirectory); 524 addAttribute2(attributes, ATTR_IMPORT_SKIP_DN_VALIDATION, skipDNValidation); 525 addAttribute2(attributes, ATTR_IMPORT_IS_COMPRESSED, isCompressed); 526 addAttribute2(attributes, ATTR_IMPORT_IS_ENCRYPTED, isEncrypted); 527 addAttribute2(attributes, ATTR_IMPORT_CLEAR_BACKEND, clearBackend); 528 } 529 530 private void addAttribute(List<RawAttribute> attributes, String attrName, String value) 531 { 532 if (value != null) 533 { 534 attributes.add(new LDAPAttribute(attrName, value)); 535 } 536 } 537 538 private void addAttribute2(List<RawAttribute> attributes, String attrName, Argument arg) 539 { 540 final String value = arg.getValue(); 541 if (value != null && !value.equals(arg.getDefaultValue())) 542 { 543 attributes.add(new LDAPAttribute(attrName, value)); 544 } 545 } 546 547 private void addAttribute(List<RawAttribute> attributes, String attrName, List<String> attrValues) 548 { 549 if (attrValues != null && !attrValues.isEmpty()) 550 { 551 attributes.add(new LDAPAttribute(attrName, attrValues)); 552 } 553 } 554 555 /** {@inheritDoc} */ 556 @Override 557 public String getTaskObjectclass() { 558 return "ds-task-import"; 559 } 560 561 /** {@inheritDoc} */ 562 @Override 563 public Class<?> getTaskClass() { 564 return ImportTask.class; 565 } 566 567 /** {@inheritDoc} */ 568 @Override 569 protected int processLocal(boolean initializeServer, 570 PrintStream out, 571 PrintStream err) { 572 573 574 // Perform the initial bootstrap of the Directory Server and process the 575 // configuration. 576 DirectoryServer directoryServer = DirectoryServer.getInstance(); 577 if (initializeServer) 578 { 579 try 580 { 581 DirectoryServer.bootstrapClient(); 582 DirectoryServer.initializeJMX(); 583 } 584 catch (Exception e) 585 { 586 printWrappedText(err, ERR_SERVER_BOOTSTRAP_ERROR.get(getExceptionMessage(e))); 587 return 1; 588 } 589 590 try 591 { 592 directoryServer.initializeConfiguration(configClass.getValue(), 593 configFile.getValue()); 594 } 595 catch (InitializationException ie) 596 { 597 printWrappedText(err, ERR_CANNOT_LOAD_CONFIG.get(ie.getMessage())); 598 return 1; 599 } 600 catch (Exception e) 601 { 602 printWrappedText(err, ERR_CANNOT_LOAD_CONFIG.get(getExceptionMessage(e))); 603 return 1; 604 } 605 606 607 608 // Initialize the Directory Server schema elements. 609 try 610 { 611 directoryServer.initializeSchema(); 612 } 613 catch (Exception e) 614 { 615 printWrappedText(err, ERR_CANNOT_LOAD_SCHEMA.get(getMessage(e))); 616 return 1; 617 } 618 619 620 // Initialize the Directory Server core configuration. 621 try 622 { 623 CoreConfigManager coreConfigManager = new CoreConfigManager(directoryServer.getServerContext()); 624 coreConfigManager.initializeCoreConfig(); 625 } 626 catch (Exception e) 627 { 628 printWrappedText(err, ERR_CANNOT_INITIALIZE_CORE_CONFIG.get(getMessage(e))); 629 return 1; 630 } 631 632 633 // Initialize the Directory Server crypto manager. 634 try 635 { 636 directoryServer.initializeCryptoManager(); 637 } 638 catch (Exception e) 639 { 640 printWrappedText(err, ERR_CANNOT_INITIALIZE_CRYPTO_MANAGER.get(getMessage(e))); 641 return 1; 642 } 643 644 645 if (! quietMode.isPresent()) 646 { 647 try 648 { 649 ErrorLogPublisher errorLogPublisher = 650 TextErrorLogPublisher.getToolStartupTextErrorPublisher( 651 new TextWriter.STREAM(out)); 652 ErrorLogger.getInstance().addLogPublisher(errorLogPublisher); 653 } 654 catch(Exception e) 655 { 656 err.println("Error installing the custom error logger: " + 657 stackTraceToSingleLineString(e)); 658 } 659 } 660 661 // Initialize the root DNs. 662 try 663 { 664 directoryServer.initializeRootDNConfigManager(); 665 } 666 catch (Exception e) 667 { 668 printWrappedText(err, ERR_CANNOT_INITIALIZE_ROOTDN_MANAGER.get(getMessage(e))); 669 return 1; 670 } 671 672 // Initialize the plugin manager. 673 try 674 { 675 HashSet<PluginType> pluginTypes = new HashSet<>(1); 676 directoryServer.initializePlugins(pluginTypes); 677 } 678 catch (Exception e) 679 { 680 printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PLUGINS.get(getMessage(e))); 681 return 1; 682 } 683 684 // Initialize the subentry manager. 685 try 686 { 687 directoryServer.initializeSubentryManager(); 688 } 689 catch (InitializationException ie) 690 { 691 printWrappedText(err, ERR_CANNOT_INITIALIZE_SUBENTRY_MANAGER.get(ie.getMessage())); 692 return 1; 693 } 694 695 // Initialize all the password policy information. 696 try 697 { 698 directoryServer.initializeAuthenticationPolicyComponents(); 699 } 700 catch (Exception e) 701 { 702 printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PWPOLICY.get(getMessage(e))); 703 return 1; 704 } 705 } 706 707 // Make sure that the plugin initialization is performed. 708 try 709 { 710 HashSet<PluginType> pluginTypes = new HashSet<>(1); 711 pluginTypes.add(PluginType.LDIF_IMPORT); 712 PluginConfigManager pluginConfigManager = 713 DirectoryServer.getPluginConfigManager(); 714 pluginConfigManager.initializeUserPlugins(pluginTypes); 715 } 716 catch (Exception e) 717 { 718 printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PLUGINS.get(getMessage(e))); 719 return 1; 720 } 721 722 // See if there were any user-defined sets of include/exclude attributes or 723 // filters. If so, then process them. 724 HashSet<AttributeType> excludeAttributes; 725 boolean excludeAllUserAttributes = false; 726 boolean excludeAllOperationalAttributes = false; 727 if (excludeAttributeStrings == null) 728 { 729 excludeAttributes = null; 730 } 731 else 732 { 733 excludeAttributes = new HashSet<>(); 734 for (String attrName : excludeAttributeStrings.getValues()) 735 { 736 String lowerName = attrName.toLowerCase(); 737 if(lowerName.equals("*")) 738 { 739 excludeAllUserAttributes = true; 740 } 741 else if(lowerName.equals("+")) 742 { 743 excludeAllOperationalAttributes = true; 744 } 745 else 746 { 747 excludeAttributes.add(DirectoryServer.getAttributeTypeOrDefault(lowerName, attrName)); 748 } 749 } 750 } 751 752 HashSet<AttributeType> includeAttributes; 753 boolean includeAllUserAttributes = false; 754 boolean includeAllOperationalAttributes = false; 755 if (includeAttributeStrings == null) 756 { 757 includeAttributes = null; 758 } 759 else 760 { 761 includeAttributes = new HashSet<>(); 762 for (String attrName : includeAttributeStrings.getValues()) 763 { 764 String lowerName = attrName.toLowerCase(); 765 if(lowerName.equals("*")) 766 { 767 includeAllUserAttributes = true; 768 } 769 else if(lowerName.equals("+")) 770 { 771 includeAllOperationalAttributes = true; 772 } 773 else 774 { 775 includeAttributes.add(DirectoryServer.getAttributeTypeOrDefault(lowerName, attrName)); 776 } 777 } 778 } 779 780 ArrayList<SearchFilter> excludeFilters; 781 if (excludeFilterStrings == null) 782 { 783 excludeFilters = null; 784 } 785 else 786 { 787 excludeFilters = new ArrayList<>(); 788 for (String filterString : excludeFilterStrings.getValues()) 789 { 790 try 791 { 792 excludeFilters.add(SearchFilter.createFilterFromString(filterString)); 793 } 794 catch (DirectoryException de) 795 { 796 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, de.getMessageObject()); 797 return 1; 798 } 799 catch (Exception e) 800 { 801 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, getExceptionMessage(e)); 802 return 1; 803 } 804 } 805 } 806 807 ArrayList<SearchFilter> includeFilters; 808 if (includeFilterStrings == null) 809 { 810 includeFilters = null; 811 } 812 else 813 { 814 includeFilters = new ArrayList<>(); 815 for (String filterString : includeFilterStrings.getValues()) 816 { 817 try 818 { 819 includeFilters.add(SearchFilter.createFilterFromString(filterString)); 820 } 821 catch (DirectoryException de) 822 { 823 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, de.getMessageObject()); 824 return 1; 825 } 826 catch (Exception e) 827 { 828 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, getExceptionMessage(e)); 829 return 1; 830 } 831 } 832 } 833 834 835 // Get information about the backends defined in the server. Iterate 836 // through them, finding the one backend into which the LDIF should be 837 // imported and finding backends with subordinate base DNs that should be 838 // excluded from the import. 839 Backend<?> backend = null; 840 Set<DN> defaultIncludeBranches = null; 841 Set<DN> excludeBranches = new HashSet<>(); 842 Set<DN> includeBranches = new HashSet<>(); 843 844 if (includeBranchStrings.isPresent()) 845 { 846 for (String s : includeBranchStrings.getValues()) 847 { 848 DN includeBranch; 849 try 850 { 851 includeBranch = DN.valueOf(s); 852 } 853 catch (DirectoryException de) 854 { 855 logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, de.getMessageObject()); 856 return 1; 857 } 858 catch (Exception e) 859 { 860 logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, getExceptionMessage(e)); 861 return 1; 862 } 863 864 includeBranches.add(includeBranch); 865 } 866 } 867 868 ArrayList<Backend> backendList = new ArrayList<>(); 869 ArrayList<BackendCfg> entryList = new ArrayList<>(); 870 ArrayList<List<DN>> dnList = new ArrayList<>(); 871 int code = BackendToolUtils.getBackends(backendList, entryList, dnList); 872 if (code != 0) 873 { 874 return code; 875 } 876 877 int numBackends = backendList.size(); 878 for (int i=0; i < numBackends; i++) 879 { 880 Backend<?> b = backendList.get(i); 881 882 if(backendID.isPresent()) 883 { 884 if (! backendID.getValue().equals(b.getBackendID())) 885 { 886 continue; 887 } 888 } 889 else 890 { 891 if (!useBackend(includeBranches, dnList.get(i))) 892 { 893 continue; 894 } 895 } 896 897 if (backend == null) 898 { 899 backend = b; 900 defaultIncludeBranches = new HashSet<>(dnList.get(i)); 901 } 902 else 903 { 904 logger.error(ERR_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID); 905 return 1; 906 } 907 } 908 909 if (backend == null) 910 { 911 logger.error(ERR_LDIFIMPORT_NO_BACKENDS_FOR_ID); 912 return 1; 913 } 914 else if (!backend.supports(BackendOperation.LDIF_IMPORT)) 915 { 916 logger.error(ERR_LDIFIMPORT_CANNOT_IMPORT, backendID.getValue()); 917 return 1; 918 } 919 920 for (List<DN> baseList : dnList) 921 { 922 for (DN baseDN : baseList) 923 { 924 for (DN importBase : defaultIncludeBranches) 925 { 926 if (!baseDN.equals(importBase) && baseDN.isDescendantOf(importBase)) 927 { 928 excludeBranches.add(baseDN); 929 break; 930 } 931 } 932 } 933 } 934 935 // Make sure that if the "backendID" argument was provided, no include base 936 // was included, the 937 // "clearBackend" argument was also provided if there are more then one 938 // baseDNs for the backend being imported. 939 940 if(backendID.isPresent() && !includeBranchStrings.isPresent() && 941 !append.isPresent() && 942 defaultIncludeBranches.size() > 1 && 943 !clearBackend.isPresent()) 944 { 945 StringBuilder builder = join(backend.getBaseDNs(), " / "); 946 printWrappedText(err, ERR_LDIFIMPORT_MISSING_CLEAR_BACKEND.get(builder, clearBackend.getLongIdentifier())); 947 return 1; 948 } 949 950 for (String s : excludeBranchStrings.getValues()) 951 { 952 DN excludeBranch; 953 try 954 { 955 excludeBranch = DN.valueOf(s); 956 } 957 catch (DirectoryException de) 958 { 959 logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, de.getMessageObject()); 960 return 1; 961 } 962 catch (Exception e) 963 { 964 logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, getExceptionMessage(e)); 965 return 1; 966 } 967 968 excludeBranches.add(excludeBranch); 969 } 970 971 if (! includeBranchStrings.isPresent()) 972 { 973 includeBranches = defaultIncludeBranches; 974 } 975 else 976 { 977 // Make sure the selected backend will handle all the include branches 978 for(DN includeBranch : includeBranches) 979 { 980 if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches, 981 excludeBranches)) 982 { 983 logger.error(ERR_LDIFIMPORT_INVALID_INCLUDE_BASE, includeBranch, backendID.getValue()); 984 return 1; 985 } 986 } 987 } 988 989 990 // See if the data should be read from LDIF files or generated via MakeLDIF. 991 LDIFImportConfig importConfig; 992 if (ldifFiles.isPresent()) 993 { 994 ArrayList<String> fileList = new ArrayList<>(ldifFiles.getValues()); 995 int badFileCount = 0; 996 for (String pathname : fileList) 997 { 998 File f = new File(pathname); 999 if (!f.canRead()) 1000 { 1001 logger.error(ERR_LDIFIMPORT_CANNOT_READ_FILE, pathname); 1002 badFileCount++; 1003 } 1004 } 1005 if (badFileCount > 0) 1006 { 1007 return 1; 1008 } 1009 importConfig = new LDIFImportConfig(fileList); 1010 } 1011 else 1012 { 1013 Random random = newRandom(); 1014 1015 String resourcePath = DirectoryServer.getInstanceRoot() + File.separator + 1016 PATH_MAKELDIF_RESOURCE_DIR; 1017 TemplateFile tf = new TemplateFile(resourcePath, random); 1018 1019 ArrayList<LocalizableMessage> warnings = new ArrayList<>(); 1020 try 1021 { 1022 tf.parse(templateFile.getValue(), warnings); 1023 } 1024 catch (Exception e) 1025 { 1026 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_TEMPLATE_FILE, templateFile.getValue(), e.getMessage()); 1027 return 1; 1028 } 1029 1030 importConfig = new LDIFImportConfig(tf); 1031 } 1032 1033 1034 // Create the LDIF import configuration to use when reading the LDIF. 1035 importConfig.setAppendToExistingData(append.isPresent()); 1036 importConfig.setReplaceExistingEntries(replaceExisting.isPresent()); 1037 importConfig.setCompressed(isCompressed.isPresent()); 1038 importConfig.setClearBackend(clearBackend.isPresent()); 1039 importConfig.setEncrypted(isEncrypted.isPresent()); 1040 importConfig.setExcludeAttributes(excludeAttributes); 1041 importConfig.setExcludeBranches(excludeBranches); 1042 importConfig.setExcludeFilters(excludeFilters); 1043 importConfig.setIncludeAttributes(includeAttributes); 1044 importConfig.setIncludeBranches(includeBranches); 1045 importConfig.setIncludeFilters(includeFilters); 1046 importConfig.setValidateSchema(!skipSchemaValidation.isPresent()); 1047 importConfig.setSkipDNValidation(skipDNValidation.isPresent()); 1048 importConfig.setTmpDirectory(tmpDirectory.getValue()); 1049 1050 try 1051 { 1052 importConfig.setThreadCount(threadCount.getIntValue()); 1053 } 1054 catch(Exception e) 1055 { 1056 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT, 1057 threadCount.getValue(), e.getMessage()); 1058 return 1; 1059 } 1060 1061 importConfig.setBufferSize(LDIF_BUFFER_SIZE); 1062 importConfig.setExcludeAllUserAttributes(excludeAllUserAttributes); 1063 importConfig.setExcludeAllOperationalAttributes(excludeAllOperationalAttributes); 1064 importConfig.setIncludeAllOpAttributes(includeAllOperationalAttributes); 1065 importConfig.setIncludeAllUserAttributes(includeAllUserAttributes); 1066 1067 // FIXME -- Should this be conditional? 1068 importConfig.setInvokeImportPlugins(true); 1069 1070 if (rejectFile != null) 1071 { 1072 try 1073 { 1074 ExistingFileBehavior existingBehavior = overwrite.isPresent() 1075 ? ExistingFileBehavior.OVERWRITE 1076 : ExistingFileBehavior.APPEND; 1077 1078 importConfig.writeRejectedEntries(rejectFile.getValue(), 1079 existingBehavior); 1080 } 1081 catch (Exception e) 1082 { 1083 logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_REJECTS_FILE, rejectFile.getValue(), getExceptionMessage(e)); 1084 return 1; 1085 } 1086 } 1087 1088 if (skipFile != null) 1089 { 1090 try 1091 { 1092 ExistingFileBehavior existingBehavior = overwrite.isPresent() 1093 ? ExistingFileBehavior.OVERWRITE 1094 : ExistingFileBehavior.APPEND; 1095 1096 importConfig.writeSkippedEntries(skipFile.getValue(), 1097 existingBehavior); 1098 } 1099 catch (Exception e) 1100 { 1101 logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_SKIP_FILE, skipFile.getValue(), getExceptionMessage(e)); 1102 return 1; 1103 } 1104 } 1105 1106 // Get the set of base DNs for the backend as an array. 1107 DN[] baseDNs = new DN[defaultIncludeBranches.size()]; 1108 defaultIncludeBranches.toArray(baseDNs); 1109 1110 1111 // Acquire an exclusive lock for the backend. 1112 try 1113 { 1114 String lockFile = LockFileManager.getBackendLockFileName(backend); 1115 StringBuilder failureReason = new StringBuilder(); 1116 if (! LockFileManager.acquireExclusiveLock(lockFile, failureReason)) 1117 { 1118 logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), failureReason); 1119 return 1; 1120 } 1121 } 1122 catch (Exception e) 1123 { 1124 logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e)); 1125 return 1; 1126 } 1127 1128 1129 // Launch the import. 1130 int retCode = 0; 1131 try 1132 { 1133 LDIFImportResult importResult = 1134 backend.importLDIF(importConfig, DirectoryServer.getInstance().getServerContext()); 1135 if (countRejects.isPresent()) 1136 { 1137 if (importResult.getEntriesRejected() > Integer.MAX_VALUE) 1138 { 1139 retCode = Integer.MAX_VALUE; 1140 } 1141 else 1142 { 1143 retCode = (int) importResult.getEntriesRejected(); 1144 } 1145 } 1146 } 1147 catch (DirectoryException de) 1148 { 1149 LocalizableMessage msg; 1150 if (de.getResultCode() == ResultCode.CONSTRAINT_VIOLATION) 1151 { 1152 msg = ERR_LDIFIMPORT_ERROR_CONSTRAINT_VIOLATION.get(); 1153 } 1154 else 1155 { 1156 msg = de.getMessageObject(); 1157 } 1158 logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg)); 1159 retCode = 1; 1160 } 1161 catch (Exception e) 1162 { 1163 logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT, getExceptionMessage(e)); 1164 retCode = 1; 1165 } 1166 1167 1168 // Release the exclusive lock on the backend. 1169 try 1170 { 1171 String lockFile = LockFileManager.getBackendLockFileName(backend); 1172 StringBuilder failureReason = new StringBuilder(); 1173 if (! LockFileManager.releaseLock(lockFile, failureReason)) 1174 { 1175 logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), failureReason); 1176 retCode = 1; 1177 } 1178 } 1179 catch (Exception e) 1180 { 1181 logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e)); 1182 retCode = 1; 1183 } 1184 1185 1186 // Clean up after the import by closing the import config. 1187 importConfig.close(); 1188 return retCode; 1189 } 1190 1191 private Object getMessage(Exception e) 1192 { 1193 try 1194 { 1195 throw e; 1196 } 1197 catch (ConfigException | InitializationException e2) 1198 { 1199 return e2.getMessage(); 1200 } 1201 catch (Exception e2) 1202 { 1203 return getExceptionMessage(e2); 1204 } 1205 } 1206 1207 private boolean useBackend(Set<DN> includeBranches, List<DN> dnlist) 1208 { 1209 for (DN baseDN : dnlist) 1210 { 1211 for (DN includeDN : includeBranches) 1212 { 1213 if (baseDN.isAncestorOf(includeDN)) 1214 { 1215 return true; 1216 } 1217 } 1218 } 1219 return false; 1220 } 1221 1222 private StringBuilder join(final DN[] baseDNs, final String separator) 1223 { 1224 final StringBuilder builder = new StringBuilder(); 1225 builder.append(baseDNs[0]); 1226 for (int i = 1; i < baseDNs.length; i++) 1227 { 1228 builder.append(separator); 1229 builder.append(baseDNs[i]); 1230 } 1231 return builder; 1232 } 1233 1234 private Random newRandom() 1235 { 1236 if (randomSeed.isPresent()) 1237 { 1238 try 1239 { 1240 return new Random(randomSeed.getIntValue()); 1241 } 1242 catch (Exception ignored) 1243 { 1244 // ignore 1245 } 1246 } 1247 return new Random(); 1248 } 1249 1250 /** {@inheritDoc} */ 1251 @Override 1252 public String getTaskId() { 1253 // NYI. 1254 return null; 1255 } 1256}