001/*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License").  You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
010 * or http://forgerock.org/license/CDDLv1.0.html.
011 * See the License for the specific language governing permissions
012 * and limitations under the License.
013 *
014 * When distributing Covered Code, include this CDDL HEADER in each
015 * file and include the License file at legal-notices/CDDLv1_0.txt.
016 * If applicable, add the following below this CDDL HEADER, with the
017 * fields enclosed by brackets "[]" replaced with your own identifying
018 * information:
019 *      Portions Copyright [yyyy] [name of copyright owner]
020 *
021 * CDDL HEADER END
022 *
023 *
024 *      Copyright 2006-2009 Sun Microsystems, Inc.
025 *      Portions Copyright 2013-2015 ForgeRock AS
026 */
027package org.opends.server.tasks;
028
029import static org.opends.messages.TaskMessages.*;
030import static org.opends.messages.ToolMessages.*;
031import static org.opends.server.config.ConfigConstants.*;
032import static org.opends.server.core.DirectoryServer.*;
033import static org.opends.server.util.StaticUtils.*;
034
035import java.io.File;
036import java.util.ArrayList;
037import java.util.Collections;
038import java.util.HashMap;
039import java.util.HashSet;
040import java.util.List;
041import java.util.Map;
042import java.util.Random;
043
044import org.forgerock.i18n.LocalizableMessage;
045import org.forgerock.i18n.slf4j.LocalizedLogger;
046import org.forgerock.opendj.ldap.ResultCode;
047import org.opends.messages.Severity;
048import org.opends.messages.TaskMessages;
049import org.opends.server.api.Backend;
050import org.opends.server.api.Backend.BackendOperation;
051import org.opends.server.api.ClientConnection;
052import org.opends.server.backends.task.Task;
053import org.opends.server.backends.task.TaskState;
054import org.opends.server.core.DirectoryServer;
055import org.opends.server.core.LockFileManager;
056import org.opends.server.tools.makeldif.TemplateFile;
057import org.opends.server.types.Attribute;
058import org.opends.server.types.AttributeType;
059import org.opends.server.types.DN;
060import org.opends.server.types.DirectoryException;
061import org.opends.server.types.Entry;
062import org.opends.server.types.ExistingFileBehavior;
063import org.opends.server.types.LDIFImportConfig;
064import org.opends.server.types.Operation;
065import org.opends.server.types.Privilege;
066import org.opends.server.types.SearchFilter;
067
068/**
069 * This class provides an implementation of a Directory Server task that can
070 * be used to import data from an LDIF file into a backend.
071 */
072public class ImportTask extends Task
073{
074  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
075
076  /** Stores mapping between configuration attribute name and its label. */
077  private static final Map<String, LocalizableMessage> argDisplayMap = new HashMap<>();
078  static
079  {
080    argDisplayMap.put(ATTR_IMPORT_LDIF_FILE, INFO_IMPORT_ARG_LDIF_FILE.get());
081    argDisplayMap.put(ATTR_IMPORT_TEMPLATE_FILE, INFO_IMPORT_ARG_TEMPLATE_FILE.get());
082    argDisplayMap.put(ATTR_IMPORT_RANDOM_SEED, INFO_IMPORT_ARG_RANDOM_SEED.get());
083    argDisplayMap.put(ATTR_IMPORT_APPEND, INFO_IMPORT_ARG_APPEND.get());
084    argDisplayMap.put(ATTR_IMPORT_REPLACE_EXISTING, INFO_IMPORT_ARG_REPLACE_EXISTING.get());
085    argDisplayMap.put(ATTR_IMPORT_BACKEND_ID, INFO_IMPORT_ARG_BACKEND_ID.get());
086    argDisplayMap.put(ATTR_IMPORT_INCLUDE_BRANCH, INFO_IMPORT_ARG_INCL_BRANCH.get());
087    argDisplayMap.put(ATTR_IMPORT_EXCLUDE_BRANCH, INFO_IMPORT_ARG_EXCL_BRANCH.get());
088    argDisplayMap.put(ATTR_IMPORT_INCLUDE_ATTRIBUTE, INFO_IMPORT_ARG_INCL_ATTR.get());
089    argDisplayMap.put(ATTR_IMPORT_EXCLUDE_ATTRIBUTE, INFO_IMPORT_ARG_EXCL_ATTR.get());
090    argDisplayMap.put(ATTR_IMPORT_INCLUDE_FILTER, INFO_IMPORT_ARG_INCL_FILTER.get());
091    argDisplayMap.put(ATTR_IMPORT_EXCLUDE_FILTER, INFO_IMPORT_ARG_EXCL_FILTER.get());
092    argDisplayMap.put(ATTR_IMPORT_REJECT_FILE, INFO_IMPORT_ARG_REJECT_FILE.get());
093    argDisplayMap.put(ATTR_IMPORT_SKIP_FILE, INFO_IMPORT_ARG_SKIP_FILE.get());
094    argDisplayMap.put(ATTR_IMPORT_OVERWRITE, INFO_IMPORT_ARG_OVERWRITE.get());
095    argDisplayMap.put(ATTR_IMPORT_SKIP_SCHEMA_VALIDATION, INFO_IMPORT_ARG_SKIP_SCHEMA_VALIDATION.get());
096    argDisplayMap.put(ATTR_IMPORT_IS_COMPRESSED, INFO_IMPORT_ARG_IS_COMPRESSED.get());
097    argDisplayMap.put(ATTR_IMPORT_IS_ENCRYPTED, INFO_IMPORT_ARG_IS_ENCRYPTED.get());
098    argDisplayMap.put(ATTR_IMPORT_CLEAR_BACKEND, INFO_IMPORT_ARG_CLEAR_BACKEND.get());
099  }
100
101
102  private boolean append;
103  private boolean isCompressed;
104  private boolean isEncrypted;
105  private boolean overwrite;
106  private boolean replaceExisting;
107  private boolean skipSchemaValidation;
108  private boolean clearBackend;
109  private boolean skipDNValidation;
110  private String tmpDirectory;
111  private int threadCount;
112  private String backendID;
113  private String rejectFile;
114  private String skipFile;
115  private ArrayList<String> excludeAttributeStrings;
116  private ArrayList<String> excludeBranchStrings;
117  private ArrayList<String> excludeFilterStrings;
118  private ArrayList<String> includeAttributeStrings;
119  private ArrayList<String> includeBranchStrings;
120  private ArrayList<String> includeFilterStrings;
121  private ArrayList<String> ldifFiles;
122  private String templateFile;
123  private int randomSeed;
124  private LDIFImportConfig importConfig;
125
126  /** {@inheritDoc} */
127  @Override
128  public LocalizableMessage getDisplayName() {
129    return INFO_TASK_IMPORT_NAME.get();
130  }
131
132  /** {@inheritDoc} */
133  @Override
134  public LocalizableMessage getAttributeDisplayName(String name) {
135    return argDisplayMap.get(name);
136  }
137
138  /** {@inheritDoc} */
139  @Override public void initializeTask() throws DirectoryException
140  {
141    // If the client connection is available, then make sure the associated
142    // client has the LDIF_IMPORT privilege.
143    Operation operation = getOperation();
144    if (operation != null)
145    {
146      ClientConnection clientConnection = operation.getClientConnection();
147      if (! clientConnection.hasPrivilege(Privilege.LDIF_IMPORT, operation))
148      {
149        LocalizableMessage message = ERR_TASK_LDIFIMPORT_INSUFFICIENT_PRIVILEGES.get();
150        throw new DirectoryException(ResultCode.INSUFFICIENT_ACCESS_RIGHTS, message);
151      }
152    }
153
154
155    Entry taskEntry = getTaskEntry();
156
157    AttributeType typeLdifFile = getAttributeTypeOrDefault(ATTR_IMPORT_LDIF_FILE);
158    AttributeType typeTemplateFile = getAttributeTypeOrDefault(ATTR_IMPORT_TEMPLATE_FILE);
159    AttributeType typeAppend = getAttributeTypeOrDefault(ATTR_IMPORT_APPEND);
160    AttributeType typeReplaceExisting = getAttributeTypeOrDefault(ATTR_IMPORT_REPLACE_EXISTING);
161    AttributeType typeBackendID = getAttributeTypeOrDefault(ATTR_IMPORT_BACKEND_ID);
162    AttributeType typeIncludeBranch = getAttributeTypeOrDefault(ATTR_IMPORT_INCLUDE_BRANCH);
163    AttributeType typeExcludeBranch = getAttributeTypeOrDefault(ATTR_IMPORT_EXCLUDE_BRANCH);
164    AttributeType typeIncludeAttribute = getAttributeTypeOrDefault(ATTR_IMPORT_INCLUDE_ATTRIBUTE);
165    AttributeType typeExcludeAttribute = getAttributeTypeOrDefault(ATTR_IMPORT_EXCLUDE_ATTRIBUTE);
166    AttributeType typeIncludeFilter = getAttributeTypeOrDefault(ATTR_IMPORT_INCLUDE_FILTER);
167    AttributeType typeExcludeFilter = getAttributeTypeOrDefault(ATTR_IMPORT_EXCLUDE_FILTER);
168    AttributeType typeRejectFile = getAttributeTypeOrDefault(ATTR_IMPORT_REJECT_FILE);
169    AttributeType typeSkipFile = getAttributeTypeOrDefault(ATTR_IMPORT_SKIP_FILE);
170    AttributeType typeOverwrite = getAttributeTypeOrDefault(ATTR_IMPORT_OVERWRITE);
171    AttributeType typeSkipSchemaValidation = getAttributeTypeOrDefault(ATTR_IMPORT_SKIP_SCHEMA_VALIDATION);
172    AttributeType typeIsCompressed = getAttributeTypeOrDefault(ATTR_IMPORT_IS_COMPRESSED);
173    AttributeType typeIsEncrypted = getAttributeTypeOrDefault(ATTR_IMPORT_IS_ENCRYPTED);
174    AttributeType typeClearBackend = getAttributeTypeOrDefault(ATTR_IMPORT_CLEAR_BACKEND);
175    AttributeType typeRandomSeed = getAttributeTypeOrDefault(ATTR_IMPORT_RANDOM_SEED);
176    AttributeType typeThreadCount = getAttributeTypeOrDefault(ATTR_IMPORT_THREAD_COUNT);
177    AttributeType typeTmpDirectory = getAttributeTypeOrDefault(ATTR_IMPORT_TMP_DIRECTORY);
178    AttributeType typeDNCheckPhase2 = getAttributeTypeOrDefault(ATTR_IMPORT_SKIP_DN_VALIDATION);
179
180    ArrayList<String> ldifFilestmp = asListOfStrings(taskEntry, typeLdifFile);
181    ldifFiles = new ArrayList<>(ldifFilestmp.size());
182    for (String s : ldifFilestmp)
183    {
184      File f = new File (s);
185      if (!f.isAbsolute())
186      {
187        f = new File(DirectoryServer.getInstanceRoot(), s);
188        try
189        {
190          s = f.getCanonicalPath();
191        }
192        catch (Exception ex)
193        {
194          s = f.getAbsolutePath();
195        }
196      }
197      ldifFiles.add(s);
198    }
199
200    templateFile = asString(taskEntry, typeTemplateFile);
201    if (templateFile != null)
202    {
203      File f = new File(templateFile);
204      if (!f.isAbsolute())
205      {
206        templateFile = new File(DirectoryServer.getInstanceRoot(), templateFile)
207            .getAbsolutePath();
208      }
209    }
210
211    append = asBoolean(taskEntry, typeAppend);
212    skipDNValidation = asBoolean(taskEntry, typeDNCheckPhase2);
213    tmpDirectory = asString(taskEntry, typeTmpDirectory);
214    replaceExisting = asBoolean(taskEntry, typeReplaceExisting);
215    backendID = asString(taskEntry, typeBackendID);
216    includeBranchStrings = asListOfStrings(taskEntry, typeIncludeBranch);
217    excludeBranchStrings = asListOfStrings(taskEntry, typeExcludeBranch);
218    includeAttributeStrings = asListOfStrings(taskEntry, typeIncludeAttribute);
219    excludeAttributeStrings = asListOfStrings(taskEntry, typeExcludeAttribute);
220    includeFilterStrings = asListOfStrings(taskEntry, typeIncludeFilter);
221    excludeFilterStrings = asListOfStrings(taskEntry, typeExcludeFilter);
222    rejectFile = asString(taskEntry, typeRejectFile);
223    skipFile = asString(taskEntry, typeSkipFile);
224    overwrite = asBoolean(taskEntry, typeOverwrite);
225    skipSchemaValidation = asBoolean(taskEntry, typeSkipSchemaValidation);
226    isCompressed = asBoolean(taskEntry, typeIsCompressed);
227    isEncrypted = asBoolean(taskEntry, typeIsEncrypted);
228    clearBackend = asBoolean(taskEntry, typeClearBackend);
229    randomSeed = asInt(taskEntry, typeRandomSeed);
230    threadCount = asInt(taskEntry, typeThreadCount);
231
232    // Make sure that either the "includeBranchStrings" argument or the
233    // "backendID" argument was provided.
234    if(includeBranchStrings.isEmpty() && backendID == null)
235    {
236      LocalizableMessage message = ERR_LDIFIMPORT_MISSING_BACKEND_ARGUMENT.get(
237          typeIncludeBranch.getNameOrOID(), typeBackendID.getNameOrOID());
238      throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
239    }
240
241    Backend<?> backend = null;
242    ArrayList<DN> defaultIncludeBranches;
243    HashSet<DN> excludeBranches = new HashSet<>(excludeBranchStrings.size());
244    HashSet<DN> includeBranches = new HashSet<>(includeBranchStrings.size());
245
246    for (String s : includeBranchStrings)
247    {
248      DN includeBranch;
249      try
250      {
251        includeBranch = DN.valueOf(s);
252      }
253      catch (DirectoryException de)
254      {
255        LocalizableMessage message = ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE.get(
256            s, de.getMessageObject());
257        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
258      }
259      catch (Exception e)
260      {
261        LocalizableMessage message = ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE.get(
262            s, getExceptionMessage(e));
263        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
264      }
265
266      includeBranches.add(includeBranch);
267    }
268    for (String s : excludeBranchStrings)
269    {
270      DN excludeBranch;
271      try
272      {
273        excludeBranch = DN.valueOf(s);
274      }
275      catch (DirectoryException de)
276      {
277        LocalizableMessage message = ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE.get(
278            s, de.getMessageObject());
279        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
280      }
281      catch (Exception e)
282      {
283        LocalizableMessage message = ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE.get(
284            s, getExceptionMessage(e));
285        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
286      }
287
288      excludeBranches.add(excludeBranch);
289    }
290
291    for (String filterString : excludeFilterStrings)
292    {
293      try
294      {
295        SearchFilter.createFilterFromString(filterString);
296      }
297      catch (DirectoryException de)
298      {
299        LocalizableMessage message = ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER.get(
300            filterString, de.getMessageObject());
301        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
302      }
303    }
304
305    for (String filterString : includeFilterStrings)
306    {
307      try
308      {
309        SearchFilter.createFilterFromString(filterString);
310      }
311      catch (DirectoryException de)
312      {
313        LocalizableMessage message = ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER.get(
314            filterString, de.getMessageObject());
315        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
316      }
317    }
318
319    if(backendID != null)
320    {
321      backend = DirectoryServer.getBackend(backendID);
322      if (backend == null)
323      {
324        LocalizableMessage message = ERR_LDIFIMPORT_NO_BACKENDS_FOR_ID.get();
325        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
326      }
327      else if (!backend.supports(BackendOperation.LDIF_IMPORT))
328      {
329        LocalizableMessage message = ERR_LDIFIMPORT_CANNOT_IMPORT.get(backendID);
330        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
331      }
332      // Make sure that if the "backendID" argument was provided, no include
333      // base was included, and the "append" option was not provided, the
334      // "clearBackend" argument was also provided if there are more then one
335      // baseDNs for the backend being imported.
336      else if(!append && includeBranchStrings.isEmpty() &&
337          backend.getBaseDNs().length > 1 && !clearBackend)
338      {
339        StringBuilder builder = new StringBuilder();
340        for(DN dn : backend.getBaseDNs())
341        {
342          builder.append(dn).append(" ");
343        }
344        LocalizableMessage message = ERR_LDIFIMPORT_MISSING_CLEAR_BACKEND.get(
345            builder, typeClearBackend.getNameOrOID());
346        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
347      }
348    }
349    else
350    {
351      // Find the backend that includes all the branches.
352      for(DN includeBranch : includeBranches)
353      {
354        Backend<?> locatedBackend = DirectoryServer.getBackend(includeBranch);
355        if(locatedBackend != null)
356        {
357          if(backend == null)
358          {
359            backend = locatedBackend;
360          }
361          else if(backend != locatedBackend)
362          {
363            // The include branches span across multiple backends.
364            LocalizableMessage message = ERR_LDIFIMPORT_INVALID_INCLUDE_BASE.get(
365                includeBranch, backend.getBackendID());
366            throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
367          }
368        }
369        else
370        {
371          // The include branch is not associated with any backend.
372          LocalizableMessage message = ERR_NO_BACKENDS_FOR_BASE.get(includeBranch);
373          throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
374        }
375      }
376    }
377
378    // Make sure the selected backend will handle all the include branches
379    defaultIncludeBranches = new ArrayList<>(backend.getBaseDNs().length);
380    Collections.addAll(defaultIncludeBranches, backend.getBaseDNs());
381
382    for(DN includeBranch : includeBranches)
383    {
384      if (!Backend.handlesEntry(includeBranch, defaultIncludeBranches, excludeBranches))
385      {
386        LocalizableMessage message = ERR_LDIFIMPORT_INVALID_INCLUDE_BASE.get(
387            includeBranch, backend.getBackendID());
388        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message);
389      }
390    }
391  }
392
393  private int asInt(Entry taskEntry, AttributeType typeRandomSeed)
394  {
395    final List<Attribute> attrList = taskEntry.getAttribute(typeRandomSeed);
396    return TaskUtils.getSingleValueInteger(attrList, 0);
397  }
398
399  private boolean asBoolean(Entry taskEntry, AttributeType typeReplaceExisting)
400  {
401    final List<Attribute> attrList = taskEntry.getAttribute(typeReplaceExisting);
402    return TaskUtils.getBoolean(attrList, false);
403  }
404
405  private String asString(Entry taskEntry, AttributeType typeBackendID)
406  {
407    final List<Attribute> attrList = taskEntry.getAttribute(typeBackendID);
408    return TaskUtils.getSingleValueString(attrList);
409  }
410
411  private ArrayList<String> asListOfStrings(Entry taskEntry, AttributeType typeExcludeBranch)
412  {
413    final List<Attribute> attrList = taskEntry.getAttribute(typeExcludeBranch);
414    return TaskUtils.getMultiValueString(attrList);
415  }
416
417  /** {@inheritDoc} */
418  @Override
419  public void interruptTask(TaskState interruptState, LocalizableMessage interruptReason)
420  {
421    if (TaskState.STOPPED_BY_ADMINISTRATOR.equals(interruptState) && importConfig != null)
422    {
423      addLogMessage(Severity.INFORMATION, TaskMessages.INFO_TASK_STOPPED_BY_ADMIN.get(
424      interruptReason));
425      setTaskInterruptState(interruptState);
426      importConfig.cancel();
427    }
428  }
429
430  /** {@inheritDoc} */
431  @Override
432  public boolean isInterruptable()
433  {
434    return true;
435  }
436
437  /** {@inheritDoc} */
438  @Override
439  protected TaskState runTask()
440  {
441    // See if there were any user-defined sets of include/exclude attributes or
442    // filters.  If so, then process them.
443    HashSet<AttributeType> excludeAttributes = toAttributeTypes(excludeAttributeStrings);
444    HashSet<AttributeType> includeAttributes = toAttributeTypes(includeAttributeStrings);
445
446    ArrayList<SearchFilter> excludeFilters = new ArrayList<>(excludeFilterStrings.size());
447    for (String filterString : excludeFilterStrings)
448    {
449      try
450      {
451        excludeFilters.add(SearchFilter.createFilterFromString(filterString));
452      }
453      catch (DirectoryException de)
454      {
455        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, de.getMessageObject());
456        return TaskState.STOPPED_BY_ERROR;
457      }
458    }
459
460    ArrayList<SearchFilter> includeFilters = new ArrayList<>(includeFilterStrings.size());
461    for (String filterString : includeFilterStrings)
462    {
463      try
464      {
465        includeFilters.add(SearchFilter.createFilterFromString(filterString));
466      }
467      catch (DirectoryException de)
468      {
469        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, de.getMessageObject());
470        return TaskState.STOPPED_BY_ERROR;
471      }
472    }
473
474
475    // Get the backend into which the LDIF should be imported.
476    Backend<?> backend = null;
477    HashSet<DN> defaultIncludeBranches;
478    HashSet<DN> excludeBranches = new HashSet<>(excludeBranchStrings.size());
479    HashSet<DN> includeBranches = new HashSet<>(includeBranchStrings.size());
480
481    for (String s : includeBranchStrings)
482    {
483      DN includeBranch;
484      try
485      {
486        includeBranch = DN.valueOf(s);
487      }
488      catch (DirectoryException de)
489      {
490        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, de.getMessageObject());
491        return TaskState.STOPPED_BY_ERROR;
492      }
493      catch (Exception e)
494      {
495        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, getExceptionMessage(e));
496        return TaskState.STOPPED_BY_ERROR;
497      }
498
499      includeBranches.add(includeBranch);
500    }
501
502    if(backendID != null)
503    {
504      backend = DirectoryServer.getBackend(backendID);
505
506      if (backend == null)
507      {
508        logger.error(ERR_LDIFIMPORT_NO_BACKENDS_FOR_ID);
509        return TaskState.STOPPED_BY_ERROR;
510      }
511      else if (!backend.supports(BackendOperation.LDIF_IMPORT))
512      {
513        logger.error(ERR_LDIFIMPORT_CANNOT_IMPORT, backendID);
514        return TaskState.STOPPED_BY_ERROR;
515      }
516      // Make sure that if the "backendID" argument was provided, no include
517      // base was included, and the "append" option was not provided, the
518      // "clearBackend" argument was also provided if there are more then one
519      // baseDNs for the backend being imported.
520      else if(!append && includeBranches.isEmpty() &&
521          backend.getBaseDNs().length > 1 && !clearBackend)
522      {
523        StringBuilder builder = new StringBuilder();
524        builder.append(backend.getBaseDNs()[0]);
525        for(int i = 1; i < backend.getBaseDNs().length; i++)
526        {
527          builder.append(" / ");
528          builder.append(backend.getBaseDNs()[i]);
529        }
530        logger.error(ERR_LDIFIMPORT_MISSING_CLEAR_BACKEND, builder, ATTR_IMPORT_CLEAR_BACKEND);
531        return TaskState.STOPPED_BY_ERROR;
532      }
533    }
534    else
535    {
536      // Find the backend that includes all the branches.
537      for(DN includeBranch : includeBranches)
538      {
539        Backend<?> locatedBackend = DirectoryServer.getBackend(includeBranch);
540        if(locatedBackend != null)
541        {
542          if(backend == null)
543          {
544            backend = locatedBackend;
545          }
546          else if(backend != locatedBackend)
547          {
548            // The include branches span across multiple backends.
549            logger.error(ERR_LDIFIMPORT_INVALID_INCLUDE_BASE, includeBranch, backend.getBackendID());
550            return TaskState.STOPPED_BY_ERROR;
551          }
552        }
553      }
554    }
555
556    // Find backends with subordinate base DNs that should be excluded from the import.
557    defaultIncludeBranches = new HashSet<>(backend.getBaseDNs().length);
558    Collections.addAll(defaultIncludeBranches, backend.getBaseDNs());
559
560    if (backend.getSubordinateBackends() != null)
561    {
562      for (Backend<?> subBackend : backend.getSubordinateBackends())
563      {
564        for (DN baseDN : subBackend.getBaseDNs())
565        {
566          for (DN importBase : defaultIncludeBranches)
567          {
568            if (!baseDN.equals(importBase) && baseDN.isDescendantOf(importBase))
569            {
570              excludeBranches.add(baseDN);
571              break;
572            }
573          }
574        }
575      }
576    }
577
578    for (String s : excludeBranchStrings)
579    {
580      DN excludeBranch;
581      try
582      {
583        excludeBranch = DN.valueOf(s);
584      }
585      catch (DirectoryException de)
586      {
587        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, de.getMessageObject());
588        return TaskState.STOPPED_BY_ERROR;
589      }
590      catch (Exception e)
591      {
592        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, getExceptionMessage(e));
593        return TaskState.STOPPED_BY_ERROR;
594      }
595
596      excludeBranches.add(excludeBranch);
597    }
598
599    if (includeBranchStrings.isEmpty())
600    {
601      includeBranches = defaultIncludeBranches;
602    }
603    else
604    {
605      // Make sure the selected backend will handle all the include branches
606      for (DN includeBranch : includeBranches)
607      {
608        if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches,
609                                   excludeBranches))
610        {
611          logger.error(ERR_LDIFIMPORT_INVALID_INCLUDE_BASE, includeBranch, backend.getBackendID());
612          return TaskState.STOPPED_BY_ERROR;
613        }
614      }
615    }
616
617    // Create the LDIF import configuration to use when reading the LDIF.
618    if (templateFile != null)
619    {
620      Random random;
621      try
622      {
623        random = new Random(randomSeed);
624      }
625      catch (Exception e)
626      {
627        random = new Random();
628      }
629
630      String resourcePath = DirectoryServer.getInstanceRoot() + File.separator +
631                            PATH_MAKELDIF_RESOURCE_DIR;
632      TemplateFile tf = new TemplateFile(resourcePath, random);
633
634      ArrayList<LocalizableMessage> warnings = new ArrayList<>();
635      try
636      {
637        tf.parse(templateFile, warnings);
638      }
639      catch (Exception e)
640      {
641        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_TEMPLATE_FILE, templateFile, e.getMessage());
642        return TaskState.STOPPED_BY_ERROR;
643      }
644
645      importConfig = new LDIFImportConfig(tf);
646    }
647    else
648    {
649      ArrayList<String> fileList = new ArrayList<>(ldifFiles);
650      importConfig = new LDIFImportConfig(fileList);
651    }
652    if(tmpDirectory == null)
653    {
654      tmpDirectory = "import-tmp";
655    }
656    importConfig.setAppendToExistingData(append);
657    importConfig.setReplaceExistingEntries(replaceExisting);
658    importConfig.setCompressed(isCompressed);
659    importConfig.setEncrypted(isEncrypted);
660    importConfig.setClearBackend(clearBackend);
661    importConfig.setExcludeAttributes(excludeAttributes);
662    importConfig.setExcludeBranches(excludeBranches);
663    importConfig.setExcludeFilters(excludeFilters);
664    importConfig.setIncludeAttributes(includeAttributes);
665    importConfig.setIncludeBranches(includeBranches);
666    importConfig.setIncludeFilters(includeFilters);
667    importConfig.setValidateSchema(!skipSchemaValidation);
668    importConfig.setSkipDNValidation(skipDNValidation);
669    importConfig.setTmpDirectory(tmpDirectory);
670    importConfig.setThreadCount(threadCount);
671
672    // FIXME -- Should this be conditional?
673    importConfig.setInvokeImportPlugins(true);
674
675    if (rejectFile != null)
676    {
677      try
678      {
679        ExistingFileBehavior existingBehavior =
680            overwrite ? ExistingFileBehavior.OVERWRITE : ExistingFileBehavior.APPEND;
681
682        importConfig.writeRejectedEntries(rejectFile, existingBehavior);
683      }
684      catch (Exception e)
685      {
686        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_REJECTS_FILE, rejectFile, getExceptionMessage(e));
687        return TaskState.STOPPED_BY_ERROR;
688      }
689    }
690
691    if (skipFile != null)
692    {
693      try
694      {
695        ExistingFileBehavior existingBehavior =
696            overwrite ? ExistingFileBehavior.OVERWRITE : ExistingFileBehavior.APPEND;
697        importConfig.writeSkippedEntries(skipFile, existingBehavior);
698      }
699      catch (Exception e)
700      {
701        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_SKIP_FILE, skipFile, getExceptionMessage(e));
702        return TaskState.STOPPED_BY_ERROR;
703      }
704    }
705
706    // Get the set of base DNs for the backend as an array.
707    DN[] baseDNs = new DN[defaultIncludeBranches.size()];
708    defaultIncludeBranches.toArray(baseDNs);
709
710    // Notify the task listeners that an import is going to start
711    // this must be done before disabling the backend to allow
712    // listeners to get access to the backend configuration
713    // and to take appropriate actions.
714    DirectoryServer.notifyImportBeginning(backend, importConfig);
715
716    // Disable the backend.
717    try
718    {
719      TaskUtils.disableBackend(backend.getBackendID());
720    }
721    catch (DirectoryException e)
722    {
723      logger.traceException(e);
724
725      logger.error(e.getMessageObject());
726      return TaskState.STOPPED_BY_ERROR;
727    }
728
729
730    try
731    {
732      // Acquire an exclusive lock for the backend.
733      try
734      {
735        String lockFile = LockFileManager.getBackendLockFileName(backend);
736        StringBuilder failureReason = new StringBuilder();
737        if (! LockFileManager.acquireExclusiveLock(lockFile, failureReason))
738        {
739          logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), failureReason);
740          return TaskState.STOPPED_BY_ERROR;
741        }
742      }
743      catch (Exception e)
744      {
745        logger.traceException(e);
746
747        logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
748        return TaskState.STOPPED_BY_ERROR;
749      }
750
751
752      // Launch the import.
753      try
754      {
755        backend.importLDIF(importConfig, DirectoryServer.getInstance().getServerContext());
756      }
757      catch (DirectoryException de)
758      {
759        logger.traceException(de);
760
761        DirectoryServer.notifyImportEnded(backend, importConfig, false);
762        LocalizableMessage msg;
763        if (de.getResultCode() == ResultCode.CONSTRAINT_VIOLATION)
764        {
765          msg = ERR_LDIFIMPORT_ERROR_CONSTRAINT_VIOLATION.get();
766        }
767        else
768        {
769          msg = de.getMessageObject();
770        }
771        logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg));
772        return TaskState.STOPPED_BY_ERROR;
773      }
774      catch (Exception e)
775      {
776        logger.traceException(e);
777
778        DirectoryServer.notifyImportEnded(backend, importConfig, false);
779        logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT, getExceptionMessage(e));
780        return TaskState.STOPPED_BY_ERROR;
781      }
782      finally
783      {
784        // Release the exclusive lock on the backend.
785        try
786        {
787          String lockFile = LockFileManager.getBackendLockFileName(backend);
788          StringBuilder failureReason = new StringBuilder();
789          if (! LockFileManager.releaseLock(lockFile, failureReason))
790          {
791            logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), failureReason);
792            return TaskState.COMPLETED_WITH_ERRORS;
793          }
794        }
795        catch (Exception e)
796        {
797          logger.traceException(e);
798
799          logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
800          return TaskState.COMPLETED_WITH_ERRORS;
801        }
802
803      }
804    }
805    finally
806    {
807      // Enable the backend.
808      try
809      {
810        TaskUtils.enableBackend(backend.getBackendID());
811        // It is necessary to retrieve the backend structure again
812        // because disabling and enabling it again may have resulted
813        // in a new backend being registered to the server.
814        backend = DirectoryServer.getBackend(backend.getBackendID());
815      }
816      catch (DirectoryException e)
817      {
818        logger.traceException(e);
819
820        logger.error(e.getMessageObject());
821        return TaskState.STOPPED_BY_ERROR;
822      }
823      DirectoryServer.notifyImportEnded(backend, importConfig, true);
824    }
825
826
827    // Clean up after the import by closing the import config.
828    importConfig.close();
829    return getFinalTaskState();
830  }
831
832  private HashSet<AttributeType> toAttributeTypes(ArrayList<String> attrNames)
833  {
834    final HashSet<AttributeType> attrTypes = new HashSet<>(attrNames.size());
835    for (String attrName : attrNames)
836    {
837      attrTypes.add(DirectoryServer.getAttributeTypeOrDefault(attrName.toLowerCase(), attrName));
838    }
839    return attrTypes;
840  }
841}