001/*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License").  You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
010 * or http://forgerock.org/license/CDDLv1.0.html.
011 * See the License for the specific language governing permissions
012 * and limitations under the License.
013 *
014 * When distributing Covered Code, include this CDDL HEADER in each
015 * file and include the License file at legal-notices/CDDLv1_0.txt.
016 * If applicable, add the following below this CDDL HEADER, with the
017 * fields enclosed by brackets "[]" replaced with your own identifying
018 * information:
019 *      Portions Copyright [yyyy] [name of copyright owner]
020 *
021 * CDDL HEADER END
022 *
023 *
024 *      Copyright 2007-2010 Sun Microsystems, Inc.
025 *      Portions Copyright 2013-2015 ForgeRock AS
026 */
027package org.opends.server.backends.jeb;
028
029import static com.sleepycat.je.EnvironmentConfig.*;
030
031import static org.forgerock.util.Reject.*;
032import static org.opends.messages.BackendMessages.*;
033import static org.opends.messages.UtilityMessages.*;
034import static org.opends.server.backends.jeb.ConfigurableEnvironment.*;
035import static org.opends.server.util.ServerConstants.*;
036import static org.opends.server.util.StaticUtils.*;
037
038import java.io.File;
039import java.io.FileFilter;
040import java.io.IOException;
041import java.nio.file.Files;
042import java.nio.file.Path;
043import java.util.ArrayList;
044import java.util.Collections;
045import java.util.LinkedHashMap;
046import java.util.List;
047import java.util.ListIterator;
048import java.util.Map;
049import java.util.NoSuchElementException;
050import java.util.Set;
051import java.util.SortedSet;
052import java.util.concurrent.ExecutionException;
053import java.util.concurrent.TimeUnit;
054import java.util.concurrent.atomic.AtomicInteger;
055import java.util.logging.Level;
056
057import org.forgerock.i18n.LocalizableMessage;
058import org.forgerock.i18n.slf4j.LocalizedLogger;
059import org.forgerock.opendj.config.server.ConfigChangeResult;
060import org.forgerock.opendj.config.server.ConfigException;
061import org.forgerock.opendj.ldap.ConditionResult;
062import org.forgerock.opendj.ldap.ResultCode;
063import org.forgerock.util.Reject;
064import org.opends.server.admin.server.ConfigurationChangeListener;
065import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn;
066import org.opends.server.admin.std.server.LocalDBBackendCfg;
067import org.opends.server.api.AlertGenerator;
068import org.opends.server.api.Backend;
069import org.opends.server.api.Backupable;
070import org.opends.server.api.DiskSpaceMonitorHandler;
071import org.opends.server.api.MonitorProvider;
072import org.opends.server.backends.RebuildConfig;
073import org.opends.server.backends.VerifyConfig;
074import org.opends.server.backends.pluggable.spi.StorageStatus;
075import org.opends.server.core.AddOperation;
076import org.opends.server.core.DeleteOperation;
077import org.opends.server.core.DirectoryServer;
078import org.opends.server.core.ModifyDNOperation;
079import org.opends.server.core.ModifyOperation;
080import org.opends.server.core.SearchOperation;
081import org.opends.server.core.ServerContext;
082import org.opends.server.extensions.DiskSpaceMonitor;
083import org.opends.server.types.AttributeType;
084import org.opends.server.types.BackupConfig;
085import org.opends.server.types.BackupDirectory;
086import org.opends.server.types.CanceledOperationException;
087import org.opends.server.types.DN;
088import org.opends.server.types.DirectoryException;
089import org.opends.server.types.Entry;
090import org.opends.server.types.IdentifiedException;
091import org.opends.server.types.IndexType;
092import org.opends.server.types.InitializationException;
093import org.opends.server.types.LDIFExportConfig;
094import org.opends.server.types.LDIFImportConfig;
095import org.opends.server.types.LDIFImportResult;
096import org.opends.server.types.Operation;
097import org.opends.server.types.Privilege;
098import org.opends.server.types.RestoreConfig;
099import org.opends.server.util.BackupManager;
100import org.opends.server.util.CollectionUtils;
101import org.opends.server.util.RuntimeInformation;
102
103import com.sleepycat.je.DatabaseException;
104import com.sleepycat.je.Durability;
105import com.sleepycat.je.EnvironmentConfig;
106import com.sleepycat.je.EnvironmentFailureException;
107
108/**
109 * This is an implementation of a Directory Server Backend which stores entries
110 * locally in a Berkeley DB JE database.
111 */
112public class BackendImpl extends Backend<LocalDBBackendCfg>
113    implements ConfigurationChangeListener<LocalDBBackendCfg>, AlertGenerator,
114    DiskSpaceMonitorHandler, Backupable
115{
116  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
117
118  /** The configuration of this JE backend. */
119  private LocalDBBackendCfg cfg;
120  /** The root JE container to use for this backend. */
121  private RootContainer rootContainer;
122  /** A count of the total operation threads currently in the backend. */
123  private final AtomicInteger threadTotalCount = new AtomicInteger(0);
124  /** A count of the write operation threads currently in the backend. */
125  private final AtomicInteger threadWriteCount = new AtomicInteger(0);
126  /** The base DNs defined for this backend instance. */
127  private DN[] baseDNs;
128
129  private MonitorProvider<?> rootContainerMonitor;
130  private DiskSpaceMonitor diskMonitor;
131  private StorageStatus storageStatus = StorageStatus.working();
132
133  /** The controls supported by this backend. */
134  private static final Set<String> supportedControls = CollectionUtils.newHashSet(
135      OID_SUBTREE_DELETE_CONTROL,
136      OID_PAGED_RESULTS_CONTROL,
137      OID_MANAGE_DSAIT_CONTROL,
138      OID_SERVER_SIDE_SORT_REQUEST_CONTROL,
139      OID_VLV_REQUEST_CONTROL);
140
141  /** Begin a Backend API method that reads the database. */
142  private void readerBegin()
143  {
144    threadTotalCount.getAndIncrement();
145  }
146
147  /** End a Backend API method that reads the database. */
148  private void readerEnd()
149  {
150    threadTotalCount.getAndDecrement();
151  }
152
153  /** Begin a Backend API method that writes the database. */
154  private void writerBegin()
155  {
156    threadTotalCount.getAndIncrement();
157    threadWriteCount.getAndIncrement();
158  }
159
160  /** End a Backend API method that writes the database. */
161  private void writerEnd()
162  {
163    threadWriteCount.getAndDecrement();
164    threadTotalCount.getAndDecrement();
165  }
166
167
168
169  /**
170   * Wait until there are no more threads accessing the database. It is assumed
171   * that new threads have been prevented from entering the database at the time
172   * this method is called.
173   */
174  private void waitUntilQuiescent()
175  {
176    while (threadTotalCount.get() > 0)
177    {
178      // Still have threads in the database so sleep a little
179      try
180      {
181        Thread.sleep(500);
182      }
183      catch (InterruptedException e)
184      {
185        logger.traceException(e);
186      }
187    }
188  }
189
190  /** {@inheritDoc} */
191  @Override
192  public void configureBackend(LocalDBBackendCfg cfg, ServerContext serverContext) throws ConfigException
193  {
194    Reject.ifNull(cfg);
195
196    this.cfg = cfg;
197    baseDNs = this.cfg.getBaseDN().toArray(new DN[0]);
198    diskMonitor = serverContext.getDiskSpaceMonitor();
199  }
200
201  /** {@inheritDoc} */
202  @Override
203  public void openBackend()
204      throws ConfigException, InitializationException
205  {
206    if (mustOpenRootContainer())
207    {
208      rootContainer = initializeRootContainer(parseConfigEntry(cfg));
209    }
210
211    // Preload the database cache.
212    rootContainer.preload(cfg.getPreloadTimeLimit());
213
214    try
215    {
216      // Log an informational message about the number of entries.
217      logger.info(NOTE_BACKEND_STARTED, cfg.getBackendId(), rootContainer.getEntryCount());
218    }
219    catch(DatabaseException databaseException)
220    {
221      logger.traceException(databaseException);
222      throw new InitializationException(
223          WARN_GET_ENTRY_COUNT_FAILED.get(databaseException.getMessage()), databaseException);
224    }
225
226    for (DN dn : cfg.getBaseDN())
227    {
228      try
229      {
230        DirectoryServer.registerBaseDN(dn, this, false);
231      }
232      catch (Exception e)
233      {
234        logger.traceException(e);
235        throw new InitializationException(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(dn, e), e);
236      }
237    }
238
239    // Register a monitor provider for the environment.
240    rootContainerMonitor = rootContainer.getMonitorProvider();
241    DirectoryServer.registerMonitorProvider(rootContainerMonitor);
242
243    // Register as disk space monitor handler
244    diskMonitor.registerMonitoredDirectory(getBackendID(), getDirectory(), cfg.getDiskLowThreshold(),
245        cfg.getDiskFullThreshold(), this);
246
247    //Register as an AlertGenerator.
248    DirectoryServer.registerAlertGenerator(this);
249    // Register this backend as a change listener.
250    cfg.addLocalDBChangeListener(this);
251  }
252
253  /** {@inheritDoc} */
254  @Override
255  public File getDirectory()
256  {
257    File parentDirectory = getFileForPath(cfg.getDBDirectory());
258    return new File(parentDirectory, cfg.getBackendId());
259  }
260
261  /** {@inheritDoc} */
262  @Override
263  public void closeBackend()
264  {
265    cfg.removeLocalDBChangeListener(this);
266
267    // Deregister our base DNs.
268    for (DN dn : rootContainer.getBaseDNs())
269    {
270      try
271      {
272        DirectoryServer.deregisterBaseDN(dn);
273      }
274      catch (Exception e)
275      {
276        logger.traceException(e);
277      }
278    }
279
280    DirectoryServer.deregisterMonitorProvider(rootContainerMonitor);
281    diskMonitor.deregisterMonitoredDirectory(getDirectory(), this);
282    // We presume the server will prevent more operations coming into this
283    // backend, but there may be existing operations already in the
284    // backend. We need to wait for them to finish.
285    waitUntilQuiescent();
286
287    // Close the database.
288    try
289    {
290      rootContainer.close();
291      rootContainer = null;
292    }
293    catch (DatabaseException e)
294    {
295      logger.traceException(e);
296      logger.error(ERR_DATABASE_EXCEPTION, e.getMessage());
297    }
298
299    DirectoryServer.deregisterAlertGenerator(this);
300
301    // Make sure the thread counts are zero for next initialization.
302    threadTotalCount.set(0);
303    threadWriteCount.set(0);
304
305    // Log an informational message.
306    logger.info(NOTE_BACKEND_OFFLINE, cfg.getBackendId());
307  }
308
309
310
311  /** {@inheritDoc} */
312  @Override
313  public boolean isIndexed(AttributeType attributeType, IndexType indexType)
314  {
315    try
316    {
317      EntryContainer ec = rootContainer.getEntryContainer(baseDNs[0]);
318      AttributeIndex ai = ec.getAttributeIndex(attributeType);
319      if (ai == null)
320      {
321        return false;
322      }
323
324      Set<LocalDBIndexCfgDefn.IndexType> indexTypes =
325           ai.getConfiguration().getIndexType();
326      switch (indexType)
327      {
328        case PRESENCE:
329          return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.PRESENCE);
330
331        case EQUALITY:
332          return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.EQUALITY);
333
334        case SUBSTRING:
335        case SUBINITIAL:
336        case SUBANY:
337        case SUBFINAL:
338          return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING);
339
340        case GREATER_OR_EQUAL:
341        case LESS_OR_EQUAL:
342          return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.ORDERING);
343
344        case APPROXIMATE:
345          return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE);
346
347        default:
348          return false;
349      }
350    }
351    catch (Exception e)
352    {
353      logger.traceException(e);
354
355      return false;
356    }
357  }
358
359  /** {@inheritDoc} */
360  @Override
361  public boolean supports(BackendOperation backendOperation)
362  {
363    // it supports all the operations so far
364    return true;
365  }
366
367  /** {@inheritDoc} */
368  @Override
369  public Set<String> getSupportedFeatures()
370  {
371    return Collections.emptySet();
372  }
373
374  /** {@inheritDoc} */
375  @Override
376  public Set<String> getSupportedControls()
377  {
378    return supportedControls;
379  }
380
381  /** {@inheritDoc} */
382  @Override
383  public DN[] getBaseDNs()
384  {
385    return baseDNs;
386  }
387
388  /** {@inheritDoc} */
389  @Override
390  public long getEntryCount()
391  {
392    if (rootContainer != null)
393    {
394      try
395      {
396        return rootContainer.getEntryCount();
397      }
398      catch (Exception e)
399      {
400        logger.traceException(e);
401      }
402    }
403
404    return -1;
405  }
406
407
408
409  /** {@inheritDoc} */
410  @Override
411  public ConditionResult hasSubordinates(DN entryDN)
412         throws DirectoryException
413  {
414    long ret = numSubordinates(entryDN, false);
415    if(ret < 0)
416    {
417      return ConditionResult.UNDEFINED;
418    }
419    return ConditionResult.valueOf(ret != 0);
420  }
421
422  /** {@inheritDoc} */
423  @Override
424  public long getNumberOfEntriesInBaseDN(DN baseDN) throws DirectoryException {
425    checkNotNull(baseDN, "baseDN must not be null");
426    EntryContainer ec = rootContainer.getEntryContainer(baseDN);
427    if (ec == null || !ec.getBaseDN().equals(baseDN))
428    {
429      throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, ERR_SEARCH_NO_SUCH_OBJECT.get(baseDN));
430    }
431    return numSubordinates(baseDN, true);
432  }
433
434  /** {@inheritDoc} */
435  @Override
436  public long getNumberOfChildren(DN parentDN) throws DirectoryException {
437    checkNotNull(parentDN, "parentDN must not be null");
438    return numSubordinates(parentDN, false);
439  }
440
441  private long numSubordinates(DN entryDN, boolean subtree) throws DirectoryException
442  {
443    checkRootContainerInitialized();
444    EntryContainer ec = rootContainer.getEntryContainer(entryDN);
445    if(ec == null)
446    {
447      return -1;
448    }
449
450    readerBegin();
451    ec.sharedLock.lock();
452    try
453    {
454      long count = ec.getNumSubordinates(entryDN, subtree);
455      if(count == Long.MAX_VALUE)
456      {
457        // The index entry limit has exceeded and there is no count maintained.
458        return -1;
459      }
460      return count;
461    }
462    catch (DatabaseException e)
463    {
464      logger.traceException(e);
465      throw createDirectoryException(e);
466    }
467    finally
468    {
469      ec.sharedLock.unlock();
470      readerEnd();
471    }
472  }
473
474
475
476  /** {@inheritDoc} */
477  @Override
478  public Entry getEntry(DN entryDN) throws DirectoryException
479  {
480    readerBegin();
481    checkRootContainerInitialized();
482    EntryContainer ec = rootContainer.getEntryContainer(entryDN);
483    ec.sharedLock.lock();
484    try
485    {
486      return ec.getEntry(entryDN);
487    }
488    catch (DatabaseException e)
489    {
490      logger.traceException(e);
491      throw createDirectoryException(e);
492    }
493    finally
494    {
495      ec.sharedLock.unlock();
496      readerEnd();
497    }
498  }
499
500
501
502  /** {@inheritDoc} */
503  @Override
504  public void addEntry(Entry entry, AddOperation addOperation)
505      throws DirectoryException, CanceledOperationException
506  {
507    checkDiskSpace(addOperation);
508    writerBegin();
509
510    checkRootContainerInitialized();
511    EntryContainer ec = rootContainer.getEntryContainer(entry.getName());
512    ec.sharedLock.lock();
513    try
514    {
515      ec.addEntry(entry, addOperation);
516    }
517    catch (DatabaseException e)
518    {
519      logger.traceException(e);
520      throw createDirectoryException(e);
521    }
522    finally
523    {
524      ec.sharedLock.unlock();
525      writerEnd();
526    }
527  }
528
529
530
531  /** {@inheritDoc} */
532  @Override
533  public void deleteEntry(DN entryDN, DeleteOperation deleteOperation)
534      throws DirectoryException, CanceledOperationException
535  {
536    checkDiskSpace(deleteOperation);
537    writerBegin();
538
539    checkRootContainerInitialized();
540    EntryContainer ec = rootContainer.getEntryContainer(entryDN);
541    ec.sharedLock.lock();
542    try
543    {
544      ec.deleteEntry(entryDN, deleteOperation);
545    }
546    catch (DatabaseException e)
547    {
548      logger.traceException(e);
549      throw createDirectoryException(e);
550    }
551    finally
552    {
553      ec.sharedLock.unlock();
554      writerEnd();
555    }
556  }
557
558
559
560  /** {@inheritDoc} */
561  @Override
562  public void replaceEntry(Entry oldEntry, Entry newEntry,
563      ModifyOperation modifyOperation) throws DirectoryException,
564      CanceledOperationException
565  {
566    checkDiskSpace(modifyOperation);
567    writerBegin();
568
569    checkRootContainerInitialized();
570    EntryContainer ec = rootContainer.getEntryContainer(newEntry.getName());
571    ec.sharedLock.lock();
572
573    try
574    {
575      ec.replaceEntry(oldEntry, newEntry, modifyOperation);
576    }
577    catch (DatabaseException e)
578    {
579      logger.traceException(e);
580      throw createDirectoryException(e);
581    }
582    finally
583    {
584      ec.sharedLock.unlock();
585      writerEnd();
586    }
587  }
588
589
590
591  /** {@inheritDoc} */
592  @Override
593  public void renameEntry(DN currentDN, Entry entry,
594                          ModifyDNOperation modifyDNOperation)
595      throws DirectoryException, CanceledOperationException
596  {
597    checkDiskSpace(modifyDNOperation);
598    writerBegin();
599
600    checkRootContainerInitialized();
601    EntryContainer currentContainer = rootContainer.getEntryContainer(currentDN);
602    EntryContainer container = rootContainer.getEntryContainer(entry.getName());
603
604    if (currentContainer != container)
605    {
606      // FIXME: No reason why we cannot implement a move between containers
607      // since the containers share the same database environment.
608      LocalizableMessage msg = WARN_FUNCTION_NOT_SUPPORTED.get();
609      throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, msg);
610    }
611
612    currentContainer.sharedLock.lock();
613    try
614    {
615      currentContainer.renameEntry(currentDN, entry, modifyDNOperation);
616    }
617    catch (DatabaseException e)
618    {
619      logger.traceException(e);
620      throw createDirectoryException(e);
621    }
622    finally
623    {
624      currentContainer.sharedLock.unlock();
625      writerEnd();
626    }
627  }
628
629
630
631  /** {@inheritDoc} */
632  @Override
633  public void search(SearchOperation searchOperation)
634      throws DirectoryException, CanceledOperationException
635  {
636    readerBegin();
637
638    checkRootContainerInitialized();
639    EntryContainer ec = rootContainer.getEntryContainer(searchOperation.getBaseDN());
640    ec.sharedLock.lock();
641
642    try
643    {
644      ec.search(searchOperation);
645    }
646    catch (DatabaseException e)
647    {
648      logger.traceException(e);
649      throw createDirectoryException(e);
650    }
651    finally
652    {
653      ec.sharedLock.unlock();
654      readerEnd();
655    }
656  }
657
658  private void checkRootContainerInitialized() throws DirectoryException
659  {
660    if (rootContainer == null)
661    {
662      LocalizableMessage msg = ERR_ROOT_CONTAINER_NOT_INITIALIZED.get(getBackendID());
663      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), msg);
664    }
665  }
666
667  /** {@inheritDoc} */
668  @Override
669  public void exportLDIF(LDIFExportConfig exportConfig)
670      throws DirectoryException
671  {
672    // If the backend already has the root container open, we must use the same
673    // underlying root container
674    boolean openRootContainer = mustOpenRootContainer();
675    final ResultCode errorRC = DirectoryServer.getServerErrorResultCode();
676    try
677    {
678      if (openRootContainer)
679      {
680        rootContainer = getReadOnlyRootContainer();
681      }
682
683      ExportJob exportJob = new ExportJob(exportConfig);
684      exportJob.exportLDIF(rootContainer);
685    }
686    catch (IOException ioe)
687    {
688      logger.traceException(ioe);
689      throw new DirectoryException(errorRC, ERR_EXPORT_IO_ERROR.get(ioe.getMessage()), ioe);
690    }
691    catch (DatabaseException de)
692    {
693      logger.traceException(de);
694      throw createDirectoryException(de);
695    }
696    catch (ConfigException ce)
697    {
698      throw new DirectoryException(errorRC, ce.getMessageObject(), ce);
699    }
700    catch (IdentifiedException e)
701    {
702      if (e instanceof DirectoryException)
703      {
704        throw (DirectoryException) e;
705      }
706      logger.traceException(e);
707      throw new DirectoryException(errorRC, e.getMessageObject(), e);
708    }
709    finally
710    {
711      closeTemporaryRootContainer(openRootContainer);
712    }
713  }
714
715  private boolean mustOpenRootContainer()
716  {
717    return rootContainer == null;
718  }
719
720  /** {@inheritDoc} */
721  @Override
722  public LDIFImportResult importLDIF(LDIFImportConfig importConfig, ServerContext serverContext)
723      throws DirectoryException
724  {
725    RuntimeInformation.logInfo();
726
727    // If the backend already has the root container open, we must use the same
728    // underlying root container
729    boolean openRootContainer = rootContainer == null;
730
731    // If the rootContainer is open, the backend is initialized by something else.
732    // We can't do import while the backend is online.
733    final ResultCode errorRC = DirectoryServer.getServerErrorResultCode();
734    if(!openRootContainer)
735    {
736      throw new DirectoryException(errorRC, ERR_IMPORT_BACKEND_ONLINE.get());
737    }
738
739    try
740    {
741      if (Importer.mustClearBackend(importConfig, cfg))
742      {
743        // We have the writer lock on the environment, now delete the
744        // environment and re-open it. Only do this when we are
745        // importing to all the base DNs in the backend or if the backend only
746        // have one base DN.
747        File parentDirectory = getFileForPath(cfg.getDBDirectory());
748        File backendDirectory = new File(parentDirectory, cfg.getBackendId());
749        // If the backend does not exist the import will create it.
750        if (backendDirectory.exists())
751        {
752          EnvManager.removeFiles(backendDirectory.getPath());
753        }
754      }
755
756      final EnvironmentConfig envConfig = getEnvConfigForImport();
757      final Importer importer = new Importer(importConfig, cfg, envConfig, serverContext);
758      rootContainer = initializeRootContainer(envConfig);
759      return importer.processImport(rootContainer);
760    }
761    catch (ExecutionException execEx)
762    {
763      logger.traceException(execEx);
764      if (execEx.getCause() instanceof DirectoryException)
765      {
766        throw ((DirectoryException) execEx.getCause());
767      }
768      throw new DirectoryException(errorRC, ERR_EXECUTION_ERROR.get(execEx.getMessage()));
769    }
770    catch (InterruptedException intEx)
771    {
772      logger.traceException(intEx);
773      throw new DirectoryException(errorRC, ERR_INTERRUPTED_ERROR.get(intEx.getMessage()));
774    }
775    catch (JebException | InitializationException | ConfigException e)
776    {
777      logger.traceException(e);
778      throw new DirectoryException(errorRC, e.getMessageObject());
779    }
780    finally
781    {
782      // leave the backend in the same state.
783      try
784      {
785        if (rootContainer != null)
786        {
787          long startTime = System.currentTimeMillis();
788          rootContainer.close();
789          long finishTime = System.currentTimeMillis();
790          long closeTime = (finishTime - startTime) / 1000;
791          logger.info(NOTE_IMPORT_LDIF_ROOTCONTAINER_CLOSE, closeTime);
792          rootContainer = null;
793        }
794
795        // Sync the environment to disk.
796        logger.info(NOTE_IMPORT_CLOSING_DATABASE);
797      }
798      catch (DatabaseException de)
799      {
800        logger.traceException(de);
801      }
802    }
803  }
804
805  private EnvironmentConfig getEnvConfigForImport()
806  {
807    final EnvironmentConfig envConfig = new EnvironmentConfig();
808    envConfig.setAllowCreate(true);
809    envConfig.setTransactional(false);
810    envConfig.setDurability(Durability.COMMIT_NO_SYNC);
811    envConfig.setLockTimeout(0, TimeUnit.SECONDS);
812    envConfig.setTxnTimeout(0, TimeUnit.SECONDS);
813    envConfig.setConfigParam(CLEANER_MIN_FILE_UTILIZATION,
814        String.valueOf(cfg.getDBCleanerMinUtilization()));
815    envConfig.setConfigParam(LOG_FILE_MAX,
816        String.valueOf(cfg.getDBLogFileMax()));
817    return envConfig;
818  }
819
820  /** {@inheritDoc} */
821  @Override
822  public long verifyBackend(VerifyConfig verifyConfig)
823      throws InitializationException, ConfigException, DirectoryException
824  {
825    // If the backend already has the root container open, we must use the same
826    // underlying root container
827    final boolean openRootContainer = mustOpenRootContainer();
828    try
829    {
830      if (openRootContainer)
831      {
832        rootContainer = getReadOnlyRootContainer();
833      }
834
835      VerifyJob verifyJob = new VerifyJob(verifyConfig);
836      return verifyJob.verifyBackend(rootContainer);
837    }
838    catch (DatabaseException e)
839    {
840      logger.traceException(e);
841      throw createDirectoryException(e);
842    }
843    catch (JebException e)
844    {
845      logger.traceException(e);
846      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
847                                   e.getMessageObject());
848    }
849    finally
850    {
851      closeTemporaryRootContainer(openRootContainer);
852    }
853  }
854
855
856  /** {@inheritDoc} */
857  @Override
858  public void rebuildBackend(RebuildConfig rebuildConfig, ServerContext serverContext)
859      throws InitializationException, ConfigException, DirectoryException
860  {
861    // If the backend already has the root container open, we must use the same
862    // underlying root container
863    boolean openRootContainer = mustOpenRootContainer();
864
865    /*
866     * If the rootContainer is open, the backend is initialized by something
867     * else. We can't do any rebuild of system indexes while others are using
868     * this backend.
869     */
870    final ResultCode errorRC = DirectoryServer.getServerErrorResultCode();
871    if(!openRootContainer && rebuildConfig.includesSystemIndex())
872    {
873      throw new DirectoryException(errorRC, ERR_REBUILD_BACKEND_ONLINE.get());
874    }
875
876    try
877    {
878      final EnvironmentConfig envConfig;
879      if (openRootContainer)
880      {
881        envConfig = getEnvConfigForImport();
882        rootContainer = initializeRootContainer(envConfig);
883      }
884      else
885      {
886        envConfig = parseConfigEntry(cfg);
887
888      }
889      final Importer importer = new Importer(rebuildConfig, cfg, envConfig, serverContext);
890      importer.rebuildIndexes(rootContainer);
891    }
892    catch (ExecutionException execEx)
893    {
894      logger.traceException(execEx);
895      throw new DirectoryException(errorRC, ERR_EXECUTION_ERROR.get(execEx.getMessage()));
896    }
897    catch (InterruptedException intEx)
898    {
899      logger.traceException(intEx);
900      throw new DirectoryException(errorRC, ERR_INTERRUPTED_ERROR.get(intEx.getMessage()));
901    }
902    catch (ConfigException | JebException e)
903    {
904      logger.traceException(e);
905      throw new DirectoryException(errorRC, e.getMessageObject());
906    }
907    catch (InitializationException e)
908    {
909      logger.traceException(e);
910      throw new InitializationException(e.getMessageObject());
911    }
912    finally
913    {
914      closeTemporaryRootContainer(openRootContainer);
915    }
916  }
917
918  /**
919   * If a root container was opened in the calling method method as read only,
920   * close it to leave the backend in the same state.
921   */
922  private void closeTemporaryRootContainer(boolean openRootContainer)
923  {
924    if (openRootContainer && rootContainer != null)
925    {
926      try
927      {
928        rootContainer.close();
929        rootContainer = null;
930      }
931      catch (DatabaseException e)
932      {
933        logger.traceException(e);
934      }
935    }
936  }
937
938
939  /** {@inheritDoc} */
940  @Override
941  public void createBackup(BackupConfig backupConfig) throws DirectoryException
942  {
943    new BackupManager(getBackendID()).createBackup(this, backupConfig);
944  }
945
946  /** {@inheritDoc} */
947  @Override
948  public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException
949  {
950    new BackupManager(getBackendID()).removeBackup(backupDirectory, backupID);
951  }
952
953  /** {@inheritDoc} */
954  @Override
955  public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException
956  {
957    new BackupManager(getBackendID()).restoreBackup(this, restoreConfig);
958  }
959
960  /** {@inheritDoc} */
961  @Override
962  public ListIterator<Path> getFilesToBackup() throws DirectoryException
963  {
964    return new JELogFilesIterator(getDirectory(), cfg.getBackendId());
965  }
966
967  /**
968   * Iterator on JE log files to backup.
969   * <p>
970   * The cleaner thread may delete some log files during the backup. The
971   * iterator is automatically renewed if at least one file has been deleted.
972   */
973  static class JELogFilesIterator implements ListIterator<Path>
974  {
975    /** Underlying iterator on files. */
976    private ListIterator<Path> iterator;
977
978    /** Root directory where all files are located. */
979    private final File rootDirectory;
980
981    private final String backendID;
982
983    /** Files to backup. Used to renew the iterator if necessary. */
984    private List<Path> files;
985
986    private String lastFileName = "";
987    private long lastFileSize;
988
989    JELogFilesIterator(File rootDirectory, String backendID) throws DirectoryException
990    {
991      this.rootDirectory = rootDirectory;
992      this.backendID = backendID;
993      setFiles(BackupManager.getFiles(rootDirectory, new JELogFileFilter(), backendID));
994    }
995
996    private void setFiles(List<Path> files) {
997      this.files = files;
998      Collections.sort(files);
999      if (!files.isEmpty())
1000      {
1001        Path lastFile = files.get(files.size() - 1);
1002        lastFileName = lastFile.getFileName().toString();
1003        lastFileSize = lastFile.toFile().length();
1004      }
1005      iterator = files.listIterator();
1006  }
1007
1008    /** {@inheritDoc} */
1009    @Override
1010    public boolean hasNext()
1011    {
1012      boolean hasNext = iterator.hasNext();
1013      if (!hasNext && !files.isEmpty())
1014      {
1015        try
1016        {
1017          List<Path> allFiles = BackupManager.getFiles(rootDirectory, new JELogFileFilter(), backendID);
1018          List<Path> compare = new ArrayList<>(files);
1019          compare.removeAll(allFiles);
1020          if (!compare.isEmpty())
1021          {
1022            // at least one file was deleted, the iterator must be renewed based on last file previously available
1023            List<Path> newFiles =
1024                BackupManager.getFiles(rootDirectory, new JELogFileFilter(lastFileName, lastFileSize), backendID);
1025            logger.info(NOTE_JEB_BACKUP_CLEANER_ACTIVITY.get(newFiles.size()));
1026            if (!newFiles.isEmpty())
1027            {
1028              setFiles(newFiles);
1029              hasNext = iterator.hasNext();
1030            }
1031          }
1032        }
1033        catch (DirectoryException e)
1034        {
1035          logger.error(ERR_BACKEND_LIST_FILES_TO_BACKUP.get(backendID, stackTraceToSingleLineString(e)));
1036        }
1037      }
1038      return hasNext;
1039    }
1040
1041    /** {@inheritDoc} */
1042    @Override
1043    public Path next()
1044    {
1045      if (hasNext()) {
1046        return iterator.next();
1047      }
1048      throw new NoSuchElementException();
1049    }
1050
1051    /** {@inheritDoc} */
1052    @Override
1053    public boolean hasPrevious()
1054    {
1055      return iterator.hasPrevious();
1056    }
1057
1058    /** {@inheritDoc} */
1059    @Override
1060    public Path previous()
1061    {
1062      return iterator.previous();
1063    }
1064
1065    /** {@inheritDoc} */
1066    @Override
1067    public int nextIndex()
1068    {
1069      return iterator.nextIndex();
1070    }
1071
1072    /** {@inheritDoc} */
1073    @Override
1074    public int previousIndex()
1075    {
1076      return iterator.previousIndex();
1077    }
1078
1079    /** {@inheritDoc} */
1080    @Override
1081    public void remove()
1082    {
1083      throw new UnsupportedOperationException("remove() is not implemented");
1084    }
1085
1086    /** {@inheritDoc} */
1087    @Override
1088    public void set(Path e)
1089    {
1090      throw new UnsupportedOperationException("set() is not implemented");
1091    }
1092
1093    /** {@inheritDoc} */
1094    @Override
1095    public void add(Path e)
1096    {
1097      throw new UnsupportedOperationException("add() is not implemented");
1098    }
1099
1100  }
1101
1102  /**
1103   * This class implements a FilenameFilter to detect a JE log file, possibly with a constraint
1104   * on the file name and file size.
1105   */
1106  private static class JELogFileFilter implements FileFilter {
1107
1108    private final String latestFilename;
1109    private final long latestFileSize;
1110
1111    /**
1112     * Creates the filter for log files that are newer than provided file name
1113     * or equal to provided file name and of larger size.
1114     */
1115    JELogFileFilter(String latestFilename, long latestFileSize) {
1116      this.latestFilename = latestFilename;
1117      this.latestFileSize = latestFileSize;
1118    }
1119
1120    /** Creates the filter for any JE log file. */
1121    JELogFileFilter() {
1122      this("", 0);
1123    }
1124
1125    /** {@inheritDoc} */
1126    @Override
1127    public boolean accept(File file)
1128    {
1129      String name = file.getName();
1130      int cmp = name.compareTo(latestFilename);
1131      return name.endsWith(".jdb") && (cmp > 0 || (cmp == 0 && file.length() > latestFileSize));
1132    }
1133  }
1134
1135  /** {@inheritDoc} */
1136  @Override
1137  public boolean isDirectRestore()
1138  {
1139    // restore is done in an intermediate directory
1140    return false;
1141  }
1142
1143  /** {@inheritDoc} */
1144  @Override
1145  public Path beforeRestore() throws DirectoryException
1146  {
1147    return null;
1148  }
1149
1150  /** {@inheritDoc} */
1151  @Override
1152  public void afterRestore(Path restoreDirectory, Path saveDirectory) throws DirectoryException
1153  {
1154    // intermediate directory content is moved to database directory
1155    File targetDirectory = getDirectory();
1156    recursiveDelete(targetDirectory);
1157    try
1158    {
1159      Files.move(restoreDirectory, targetDirectory.toPath());
1160    }
1161    catch(IOException e)
1162    {
1163      LocalizableMessage msg = ERR_CANNOT_RENAME_RESTORE_DIRECTORY.get(restoreDirectory, targetDirectory.getPath());
1164      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), msg);
1165    }
1166  }
1167
1168  /** {@inheritDoc} */
1169  @Override
1170  public boolean isConfigurationAcceptable(LocalDBBackendCfg config,
1171                                           List<LocalizableMessage> unacceptableReasons,
1172                                           ServerContext serverContext)
1173  {
1174    return isConfigurationChangeAcceptable(config, unacceptableReasons);
1175  }
1176
1177
1178
1179  /** {@inheritDoc} */
1180  @Override
1181  public boolean isConfigurationChangeAcceptable(
1182      LocalDBBackendCfg cfg,
1183      List<LocalizableMessage> unacceptableReasons)
1184  {
1185    // Make sure that the logging level value is acceptable.
1186    try {
1187      Level.parse(cfg.getDBLoggingLevel());
1188      return true;
1189    } catch (Exception e) {
1190      unacceptableReasons.add(ERR_JEB_INVALID_LOGGING_LEVEL.get(cfg.getDBLoggingLevel(), cfg.dn()));
1191      return false;
1192    }
1193  }
1194
1195
1196
1197  /** {@inheritDoc} */
1198  @Override
1199  public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg newCfg)
1200  {
1201    final ConfigChangeResult ccr = new ConfigChangeResult();
1202    try
1203    {
1204      if(rootContainer != null)
1205      {
1206        SortedSet<DN> newBaseDNs = newCfg.getBaseDN();
1207        DN[] newBaseDNsArray = newBaseDNs.toArray(new DN[newBaseDNs.size()]);
1208
1209        // Check for changes to the base DNs.
1210        removeDeletedBaseDNs(newBaseDNs);
1211        ConfigChangeResult failure = createNewBaseDNs(newBaseDNsArray, ccr);
1212        if (failure != null)
1213        {
1214          return failure;
1215        }
1216
1217        baseDNs = newBaseDNsArray;
1218      }
1219
1220      updateDiskMonitor(diskMonitor, newCfg);
1221
1222      // Put the new configuration in place.
1223      this.cfg = newCfg;
1224    }
1225    catch (Exception e)
1226    {
1227      ccr.addMessage(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
1228      ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
1229    }
1230    return ccr;
1231  }
1232
1233  private void updateDiskMonitor(DiskSpaceMonitor dm, LocalDBBackendCfg newCfg)
1234  {
1235    diskMonitor.registerMonitoredDirectory(getBackendID(), getDirectory(), newCfg.getDiskLowThreshold(),
1236        newCfg.getDiskFullThreshold(), this);
1237  }
1238
1239  private void removeDeletedBaseDNs(SortedSet<DN> newBaseDNs) throws DirectoryException
1240  {
1241    for (DN baseDN : cfg.getBaseDN())
1242    {
1243      if (!newBaseDNs.contains(baseDN))
1244      {
1245        // The base DN was deleted.
1246        DirectoryServer.deregisterBaseDN(baseDN);
1247        EntryContainer ec = rootContainer.unregisterEntryContainer(baseDN);
1248        ec.close();
1249        ec.delete();
1250      }
1251    }
1252  }
1253
1254  private ConfigChangeResult createNewBaseDNs(DN[] newBaseDNsArray, final ConfigChangeResult ccr)
1255  {
1256    for (DN baseDN : newBaseDNsArray)
1257    {
1258      if (!rootContainer.getBaseDNs().contains(baseDN))
1259      {
1260        try
1261        {
1262          // The base DN was added.
1263          EntryContainer ec = rootContainer.openEntryContainer(baseDN, null);
1264          rootContainer.registerEntryContainer(baseDN, ec);
1265          DirectoryServer.registerBaseDN(baseDN, this, false);
1266        }
1267        catch (Exception e)
1268        {
1269          logger.traceException(e);
1270
1271          ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
1272          ccr.addMessage(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(baseDN, e));
1273          return ccr;
1274        }
1275      }
1276    }
1277    return null;
1278  }
1279
1280  /**
1281   * Returns a handle to the JE root container currently used by this backend.
1282   * The rootContainer could be NULL if the backend is not initialized.
1283   *
1284   * @return The RootContainer object currently used by this backend.
1285   */
1286  public RootContainer getRootContainer()
1287  {
1288    return rootContainer;
1289  }
1290
1291  /**
1292   * Returns a new read-only handle to the JE root container for this backend.
1293   * The caller is responsible for closing the root container after use.
1294   *
1295   * @return The read-only RootContainer object for this backend.
1296   *
1297   * @throws  ConfigException  If an unrecoverable problem arises during
1298   *                           initialization.
1299   * @throws  InitializationException  If a problem occurs during initialization
1300   *                                   that is not related to the server
1301   *                                   configuration.
1302   */
1303  public RootContainer getReadOnlyRootContainer()
1304      throws ConfigException, InitializationException
1305  {
1306    EnvironmentConfig envConfig = parseConfigEntry(cfg);
1307
1308    envConfig.setReadOnly(true);
1309    envConfig.setAllowCreate(false);
1310    envConfig.setTransactional(false);
1311    envConfig.setConfigParam(ENV_IS_LOCKING, "true");
1312    envConfig.setConfigParam(ENV_RUN_CHECKPOINTER, "true");
1313
1314    return initializeRootContainer(envConfig);
1315  }
1316
1317  /**
1318   * Clears all the entries from the backend.  This method is for test cases
1319   * that use the JE backend.
1320   *
1321   * @throws  ConfigException  If an unrecoverable problem arises in the
1322   *                           process of performing the initialization.
1323   *
1324   * @throws  JebException     If an error occurs while removing the data.
1325   */
1326  public void clearBackend()
1327      throws ConfigException, JebException
1328  {
1329    // Determine the backend database directory.
1330    File parentDirectory = getFileForPath(cfg.getDBDirectory());
1331    File backendDirectory = new File(parentDirectory, cfg.getBackendId());
1332    EnvManager.removeFiles(backendDirectory.getPath());
1333  }
1334
1335  /**
1336   * Creates a customized DirectoryException from the DatabaseException thrown
1337   * by JE backend.
1338   *
1339   * @param  e The DatabaseException to be converted.
1340   * @return  DirectoryException created from exception.
1341   */
1342  private DirectoryException createDirectoryException(DatabaseException e) {
1343    if (e instanceof EnvironmentFailureException && !rootContainer.isValid()) {
1344      LocalizableMessage message = NOTE_BACKEND_ENVIRONMENT_UNUSABLE.get(getBackendID());
1345      logger.info(message);
1346      DirectoryServer.sendAlertNotification(DirectoryServer.getInstance(),
1347              ALERT_TYPE_BACKEND_ENVIRONMENT_UNUSABLE, message);
1348    }
1349
1350    String jeMessage = e.getMessage();
1351    if (jeMessage == null) {
1352      jeMessage = stackTraceToSingleLineString(e);
1353    }
1354    LocalizableMessage message = ERR_DATABASE_EXCEPTION.get(jeMessage);
1355    return new DirectoryException(
1356        DirectoryServer.getServerErrorResultCode(), message, e);
1357  }
1358
1359  /** {@inheritDoc} */
1360  @Override
1361  public String getClassName() {
1362    return BackendImpl.class.getName();
1363  }
1364
1365  /** {@inheritDoc} */
1366  @Override
1367  public Map<String, String> getAlerts()
1368  {
1369    Map<String, String> alerts = new LinkedHashMap<>();
1370
1371    alerts.put(ALERT_TYPE_BACKEND_ENVIRONMENT_UNUSABLE,
1372            ALERT_DESCRIPTION_BACKEND_ENVIRONMENT_UNUSABLE);
1373    alerts.put(ALERT_TYPE_DISK_SPACE_LOW,
1374            ALERT_DESCRIPTION_DISK_SPACE_LOW);
1375    alerts.put(ALERT_TYPE_DISK_FULL,
1376            ALERT_DESCRIPTION_DISK_FULL);
1377    return alerts;
1378  }
1379
1380  /** {@inheritDoc} */
1381  @Override
1382  public DN getComponentEntryDN() {
1383    return cfg.dn();
1384  }
1385
1386  private RootContainer initializeRootContainer(EnvironmentConfig envConfig)
1387          throws ConfigException, InitializationException {
1388    // Open the database environment
1389    try {
1390      RootContainer rc = new RootContainer(this, cfg);
1391      rc.open(envConfig);
1392      return rc;
1393    }
1394    catch (DatabaseException e) {
1395      logger.traceException(e);
1396      LocalizableMessage message = ERR_OPEN_ENV_FAIL.get(e.getMessage());
1397      throw new InitializationException(message, e);
1398    }
1399  }
1400
1401  /** {@inheritDoc} */
1402  @Override
1403  public void diskLowThresholdReached(File directory, long thresholdInBytes) {
1404    storageStatus = StorageStatus.lockedDown(
1405        WARN_DISK_SPACE_LOW_THRESHOLD_CROSSED.get(directory.getFreeSpace(), directory.getAbsolutePath(),
1406        thresholdInBytes, getBackendID()));
1407  }
1408
1409  /** {@inheritDoc} */
1410  @Override
1411  public void diskFullThresholdReached(File directory, long thresholdInBytes) {
1412    storageStatus = StorageStatus.unusable(
1413        WARN_DISK_SPACE_FULL_THRESHOLD_CROSSED.get(directory.getFreeSpace(), directory.getAbsolutePath(),
1414        thresholdInBytes, getBackendID()));
1415  }
1416
1417  /** {@inheritDoc} */
1418  @Override
1419  public void diskSpaceRestored(File directory, long lowThresholdInBytes, long fullThresholdInBytes) {
1420    storageStatus = StorageStatus.working();
1421  }
1422
1423  private void checkDiskSpace(Operation operation) throws DirectoryException
1424  {
1425    if(storageStatus.isUnusable() ||
1426        (storageStatus.isLockedDown()
1427            && operation != null
1428            && !operation.getClientConnection().hasPrivilege(
1429                Privilege.BYPASS_LOCKDOWN, operation)))
1430    {
1431      throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM,
1432          WARN_OUT_OF_DISK_SPACE.get());
1433    }
1434  }
1435}