001/* 002 * CDDL HEADER START 003 * 004 * The contents of this file are subject to the terms of the 005 * Common Development and Distribution License, Version 1.0 only 006 * (the "License"). You may not use this file except in compliance 007 * with the License. 008 * 009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt 010 * or http://forgerock.org/license/CDDLv1.0.html. 011 * See the License for the specific language governing permissions 012 * and limitations under the License. 013 * 014 * When distributing Covered Code, include this CDDL HEADER in each 015 * file and include the License file at legal-notices/CDDLv1_0.txt. 016 * If applicable, add the following below this CDDL HEADER, with the 017 * fields enclosed by brackets "[]" replaced with your own identifying 018 * information: 019 * Portions Copyright [yyyy] [name of copyright owner] 020 * 021 * CDDL HEADER END 022 * 023 * 024 * Copyright 2007-2010 Sun Microsystems, Inc. 025 * Portions Copyright 2013-2015 ForgeRock AS 026 */ 027package org.opends.server.backends.jeb; 028 029import static com.sleepycat.je.EnvironmentConfig.*; 030 031import static org.forgerock.util.Reject.*; 032import static org.opends.messages.BackendMessages.*; 033import static org.opends.messages.UtilityMessages.*; 034import static org.opends.server.backends.jeb.ConfigurableEnvironment.*; 035import static org.opends.server.util.ServerConstants.*; 036import static org.opends.server.util.StaticUtils.*; 037 038import java.io.File; 039import java.io.FileFilter; 040import java.io.IOException; 041import java.nio.file.Files; 042import java.nio.file.Path; 043import java.util.ArrayList; 044import java.util.Collections; 045import java.util.LinkedHashMap; 046import java.util.List; 047import java.util.ListIterator; 048import java.util.Map; 049import java.util.NoSuchElementException; 050import java.util.Set; 051import java.util.SortedSet; 052import java.util.concurrent.ExecutionException; 053import java.util.concurrent.TimeUnit; 054import java.util.concurrent.atomic.AtomicInteger; 055import java.util.logging.Level; 056 057import org.forgerock.i18n.LocalizableMessage; 058import org.forgerock.i18n.slf4j.LocalizedLogger; 059import org.forgerock.opendj.config.server.ConfigChangeResult; 060import org.forgerock.opendj.config.server.ConfigException; 061import org.forgerock.opendj.ldap.ConditionResult; 062import org.forgerock.opendj.ldap.ResultCode; 063import org.forgerock.util.Reject; 064import org.opends.server.admin.server.ConfigurationChangeListener; 065import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn; 066import org.opends.server.admin.std.server.LocalDBBackendCfg; 067import org.opends.server.api.AlertGenerator; 068import org.opends.server.api.Backend; 069import org.opends.server.api.Backupable; 070import org.opends.server.api.DiskSpaceMonitorHandler; 071import org.opends.server.api.MonitorProvider; 072import org.opends.server.backends.RebuildConfig; 073import org.opends.server.backends.VerifyConfig; 074import org.opends.server.backends.pluggable.spi.StorageStatus; 075import org.opends.server.core.AddOperation; 076import org.opends.server.core.DeleteOperation; 077import org.opends.server.core.DirectoryServer; 078import org.opends.server.core.ModifyDNOperation; 079import org.opends.server.core.ModifyOperation; 080import org.opends.server.core.SearchOperation; 081import org.opends.server.core.ServerContext; 082import org.opends.server.extensions.DiskSpaceMonitor; 083import org.opends.server.types.AttributeType; 084import org.opends.server.types.BackupConfig; 085import org.opends.server.types.BackupDirectory; 086import org.opends.server.types.CanceledOperationException; 087import org.opends.server.types.DN; 088import org.opends.server.types.DirectoryException; 089import org.opends.server.types.Entry; 090import org.opends.server.types.IdentifiedException; 091import org.opends.server.types.IndexType; 092import org.opends.server.types.InitializationException; 093import org.opends.server.types.LDIFExportConfig; 094import org.opends.server.types.LDIFImportConfig; 095import org.opends.server.types.LDIFImportResult; 096import org.opends.server.types.Operation; 097import org.opends.server.types.Privilege; 098import org.opends.server.types.RestoreConfig; 099import org.opends.server.util.BackupManager; 100import org.opends.server.util.CollectionUtils; 101import org.opends.server.util.RuntimeInformation; 102 103import com.sleepycat.je.DatabaseException; 104import com.sleepycat.je.Durability; 105import com.sleepycat.je.EnvironmentConfig; 106import com.sleepycat.je.EnvironmentFailureException; 107 108/** 109 * This is an implementation of a Directory Server Backend which stores entries 110 * locally in a Berkeley DB JE database. 111 */ 112public class BackendImpl extends Backend<LocalDBBackendCfg> 113 implements ConfigurationChangeListener<LocalDBBackendCfg>, AlertGenerator, 114 DiskSpaceMonitorHandler, Backupable 115{ 116 private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); 117 118 /** The configuration of this JE backend. */ 119 private LocalDBBackendCfg cfg; 120 /** The root JE container to use for this backend. */ 121 private RootContainer rootContainer; 122 /** A count of the total operation threads currently in the backend. */ 123 private final AtomicInteger threadTotalCount = new AtomicInteger(0); 124 /** A count of the write operation threads currently in the backend. */ 125 private final AtomicInteger threadWriteCount = new AtomicInteger(0); 126 /** The base DNs defined for this backend instance. */ 127 private DN[] baseDNs; 128 129 private MonitorProvider<?> rootContainerMonitor; 130 private DiskSpaceMonitor diskMonitor; 131 private StorageStatus storageStatus = StorageStatus.working(); 132 133 /** The controls supported by this backend. */ 134 private static final Set<String> supportedControls = CollectionUtils.newHashSet( 135 OID_SUBTREE_DELETE_CONTROL, 136 OID_PAGED_RESULTS_CONTROL, 137 OID_MANAGE_DSAIT_CONTROL, 138 OID_SERVER_SIDE_SORT_REQUEST_CONTROL, 139 OID_VLV_REQUEST_CONTROL); 140 141 /** Begin a Backend API method that reads the database. */ 142 private void readerBegin() 143 { 144 threadTotalCount.getAndIncrement(); 145 } 146 147 /** End a Backend API method that reads the database. */ 148 private void readerEnd() 149 { 150 threadTotalCount.getAndDecrement(); 151 } 152 153 /** Begin a Backend API method that writes the database. */ 154 private void writerBegin() 155 { 156 threadTotalCount.getAndIncrement(); 157 threadWriteCount.getAndIncrement(); 158 } 159 160 /** End a Backend API method that writes the database. */ 161 private void writerEnd() 162 { 163 threadWriteCount.getAndDecrement(); 164 threadTotalCount.getAndDecrement(); 165 } 166 167 168 169 /** 170 * Wait until there are no more threads accessing the database. It is assumed 171 * that new threads have been prevented from entering the database at the time 172 * this method is called. 173 */ 174 private void waitUntilQuiescent() 175 { 176 while (threadTotalCount.get() > 0) 177 { 178 // Still have threads in the database so sleep a little 179 try 180 { 181 Thread.sleep(500); 182 } 183 catch (InterruptedException e) 184 { 185 logger.traceException(e); 186 } 187 } 188 } 189 190 /** {@inheritDoc} */ 191 @Override 192 public void configureBackend(LocalDBBackendCfg cfg, ServerContext serverContext) throws ConfigException 193 { 194 Reject.ifNull(cfg); 195 196 this.cfg = cfg; 197 baseDNs = this.cfg.getBaseDN().toArray(new DN[0]); 198 diskMonitor = serverContext.getDiskSpaceMonitor(); 199 } 200 201 /** {@inheritDoc} */ 202 @Override 203 public void openBackend() 204 throws ConfigException, InitializationException 205 { 206 if (mustOpenRootContainer()) 207 { 208 rootContainer = initializeRootContainer(parseConfigEntry(cfg)); 209 } 210 211 // Preload the database cache. 212 rootContainer.preload(cfg.getPreloadTimeLimit()); 213 214 try 215 { 216 // Log an informational message about the number of entries. 217 logger.info(NOTE_BACKEND_STARTED, cfg.getBackendId(), rootContainer.getEntryCount()); 218 } 219 catch(DatabaseException databaseException) 220 { 221 logger.traceException(databaseException); 222 throw new InitializationException( 223 WARN_GET_ENTRY_COUNT_FAILED.get(databaseException.getMessage()), databaseException); 224 } 225 226 for (DN dn : cfg.getBaseDN()) 227 { 228 try 229 { 230 DirectoryServer.registerBaseDN(dn, this, false); 231 } 232 catch (Exception e) 233 { 234 logger.traceException(e); 235 throw new InitializationException(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(dn, e), e); 236 } 237 } 238 239 // Register a monitor provider for the environment. 240 rootContainerMonitor = rootContainer.getMonitorProvider(); 241 DirectoryServer.registerMonitorProvider(rootContainerMonitor); 242 243 // Register as disk space monitor handler 244 diskMonitor.registerMonitoredDirectory(getBackendID(), getDirectory(), cfg.getDiskLowThreshold(), 245 cfg.getDiskFullThreshold(), this); 246 247 //Register as an AlertGenerator. 248 DirectoryServer.registerAlertGenerator(this); 249 // Register this backend as a change listener. 250 cfg.addLocalDBChangeListener(this); 251 } 252 253 /** {@inheritDoc} */ 254 @Override 255 public File getDirectory() 256 { 257 File parentDirectory = getFileForPath(cfg.getDBDirectory()); 258 return new File(parentDirectory, cfg.getBackendId()); 259 } 260 261 /** {@inheritDoc} */ 262 @Override 263 public void closeBackend() 264 { 265 cfg.removeLocalDBChangeListener(this); 266 267 // Deregister our base DNs. 268 for (DN dn : rootContainer.getBaseDNs()) 269 { 270 try 271 { 272 DirectoryServer.deregisterBaseDN(dn); 273 } 274 catch (Exception e) 275 { 276 logger.traceException(e); 277 } 278 } 279 280 DirectoryServer.deregisterMonitorProvider(rootContainerMonitor); 281 diskMonitor.deregisterMonitoredDirectory(getDirectory(), this); 282 // We presume the server will prevent more operations coming into this 283 // backend, but there may be existing operations already in the 284 // backend. We need to wait for them to finish. 285 waitUntilQuiescent(); 286 287 // Close the database. 288 try 289 { 290 rootContainer.close(); 291 rootContainer = null; 292 } 293 catch (DatabaseException e) 294 { 295 logger.traceException(e); 296 logger.error(ERR_DATABASE_EXCEPTION, e.getMessage()); 297 } 298 299 DirectoryServer.deregisterAlertGenerator(this); 300 301 // Make sure the thread counts are zero for next initialization. 302 threadTotalCount.set(0); 303 threadWriteCount.set(0); 304 305 // Log an informational message. 306 logger.info(NOTE_BACKEND_OFFLINE, cfg.getBackendId()); 307 } 308 309 310 311 /** {@inheritDoc} */ 312 @Override 313 public boolean isIndexed(AttributeType attributeType, IndexType indexType) 314 { 315 try 316 { 317 EntryContainer ec = rootContainer.getEntryContainer(baseDNs[0]); 318 AttributeIndex ai = ec.getAttributeIndex(attributeType); 319 if (ai == null) 320 { 321 return false; 322 } 323 324 Set<LocalDBIndexCfgDefn.IndexType> indexTypes = 325 ai.getConfiguration().getIndexType(); 326 switch (indexType) 327 { 328 case PRESENCE: 329 return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.PRESENCE); 330 331 case EQUALITY: 332 return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.EQUALITY); 333 334 case SUBSTRING: 335 case SUBINITIAL: 336 case SUBANY: 337 case SUBFINAL: 338 return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING); 339 340 case GREATER_OR_EQUAL: 341 case LESS_OR_EQUAL: 342 return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.ORDERING); 343 344 case APPROXIMATE: 345 return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE); 346 347 default: 348 return false; 349 } 350 } 351 catch (Exception e) 352 { 353 logger.traceException(e); 354 355 return false; 356 } 357 } 358 359 /** {@inheritDoc} */ 360 @Override 361 public boolean supports(BackendOperation backendOperation) 362 { 363 // it supports all the operations so far 364 return true; 365 } 366 367 /** {@inheritDoc} */ 368 @Override 369 public Set<String> getSupportedFeatures() 370 { 371 return Collections.emptySet(); 372 } 373 374 /** {@inheritDoc} */ 375 @Override 376 public Set<String> getSupportedControls() 377 { 378 return supportedControls; 379 } 380 381 /** {@inheritDoc} */ 382 @Override 383 public DN[] getBaseDNs() 384 { 385 return baseDNs; 386 } 387 388 /** {@inheritDoc} */ 389 @Override 390 public long getEntryCount() 391 { 392 if (rootContainer != null) 393 { 394 try 395 { 396 return rootContainer.getEntryCount(); 397 } 398 catch (Exception e) 399 { 400 logger.traceException(e); 401 } 402 } 403 404 return -1; 405 } 406 407 408 409 /** {@inheritDoc} */ 410 @Override 411 public ConditionResult hasSubordinates(DN entryDN) 412 throws DirectoryException 413 { 414 long ret = numSubordinates(entryDN, false); 415 if(ret < 0) 416 { 417 return ConditionResult.UNDEFINED; 418 } 419 return ConditionResult.valueOf(ret != 0); 420 } 421 422 /** {@inheritDoc} */ 423 @Override 424 public long getNumberOfEntriesInBaseDN(DN baseDN) throws DirectoryException { 425 checkNotNull(baseDN, "baseDN must not be null"); 426 EntryContainer ec = rootContainer.getEntryContainer(baseDN); 427 if (ec == null || !ec.getBaseDN().equals(baseDN)) 428 { 429 throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, ERR_SEARCH_NO_SUCH_OBJECT.get(baseDN)); 430 } 431 return numSubordinates(baseDN, true); 432 } 433 434 /** {@inheritDoc} */ 435 @Override 436 public long getNumberOfChildren(DN parentDN) throws DirectoryException { 437 checkNotNull(parentDN, "parentDN must not be null"); 438 return numSubordinates(parentDN, false); 439 } 440 441 private long numSubordinates(DN entryDN, boolean subtree) throws DirectoryException 442 { 443 checkRootContainerInitialized(); 444 EntryContainer ec = rootContainer.getEntryContainer(entryDN); 445 if(ec == null) 446 { 447 return -1; 448 } 449 450 readerBegin(); 451 ec.sharedLock.lock(); 452 try 453 { 454 long count = ec.getNumSubordinates(entryDN, subtree); 455 if(count == Long.MAX_VALUE) 456 { 457 // The index entry limit has exceeded and there is no count maintained. 458 return -1; 459 } 460 return count; 461 } 462 catch (DatabaseException e) 463 { 464 logger.traceException(e); 465 throw createDirectoryException(e); 466 } 467 finally 468 { 469 ec.sharedLock.unlock(); 470 readerEnd(); 471 } 472 } 473 474 475 476 /** {@inheritDoc} */ 477 @Override 478 public Entry getEntry(DN entryDN) throws DirectoryException 479 { 480 readerBegin(); 481 checkRootContainerInitialized(); 482 EntryContainer ec = rootContainer.getEntryContainer(entryDN); 483 ec.sharedLock.lock(); 484 try 485 { 486 return ec.getEntry(entryDN); 487 } 488 catch (DatabaseException e) 489 { 490 logger.traceException(e); 491 throw createDirectoryException(e); 492 } 493 finally 494 { 495 ec.sharedLock.unlock(); 496 readerEnd(); 497 } 498 } 499 500 501 502 /** {@inheritDoc} */ 503 @Override 504 public void addEntry(Entry entry, AddOperation addOperation) 505 throws DirectoryException, CanceledOperationException 506 { 507 checkDiskSpace(addOperation); 508 writerBegin(); 509 510 checkRootContainerInitialized(); 511 EntryContainer ec = rootContainer.getEntryContainer(entry.getName()); 512 ec.sharedLock.lock(); 513 try 514 { 515 ec.addEntry(entry, addOperation); 516 } 517 catch (DatabaseException e) 518 { 519 logger.traceException(e); 520 throw createDirectoryException(e); 521 } 522 finally 523 { 524 ec.sharedLock.unlock(); 525 writerEnd(); 526 } 527 } 528 529 530 531 /** {@inheritDoc} */ 532 @Override 533 public void deleteEntry(DN entryDN, DeleteOperation deleteOperation) 534 throws DirectoryException, CanceledOperationException 535 { 536 checkDiskSpace(deleteOperation); 537 writerBegin(); 538 539 checkRootContainerInitialized(); 540 EntryContainer ec = rootContainer.getEntryContainer(entryDN); 541 ec.sharedLock.lock(); 542 try 543 { 544 ec.deleteEntry(entryDN, deleteOperation); 545 } 546 catch (DatabaseException e) 547 { 548 logger.traceException(e); 549 throw createDirectoryException(e); 550 } 551 finally 552 { 553 ec.sharedLock.unlock(); 554 writerEnd(); 555 } 556 } 557 558 559 560 /** {@inheritDoc} */ 561 @Override 562 public void replaceEntry(Entry oldEntry, Entry newEntry, 563 ModifyOperation modifyOperation) throws DirectoryException, 564 CanceledOperationException 565 { 566 checkDiskSpace(modifyOperation); 567 writerBegin(); 568 569 checkRootContainerInitialized(); 570 EntryContainer ec = rootContainer.getEntryContainer(newEntry.getName()); 571 ec.sharedLock.lock(); 572 573 try 574 { 575 ec.replaceEntry(oldEntry, newEntry, modifyOperation); 576 } 577 catch (DatabaseException e) 578 { 579 logger.traceException(e); 580 throw createDirectoryException(e); 581 } 582 finally 583 { 584 ec.sharedLock.unlock(); 585 writerEnd(); 586 } 587 } 588 589 590 591 /** {@inheritDoc} */ 592 @Override 593 public void renameEntry(DN currentDN, Entry entry, 594 ModifyDNOperation modifyDNOperation) 595 throws DirectoryException, CanceledOperationException 596 { 597 checkDiskSpace(modifyDNOperation); 598 writerBegin(); 599 600 checkRootContainerInitialized(); 601 EntryContainer currentContainer = rootContainer.getEntryContainer(currentDN); 602 EntryContainer container = rootContainer.getEntryContainer(entry.getName()); 603 604 if (currentContainer != container) 605 { 606 // FIXME: No reason why we cannot implement a move between containers 607 // since the containers share the same database environment. 608 LocalizableMessage msg = WARN_FUNCTION_NOT_SUPPORTED.get(); 609 throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, msg); 610 } 611 612 currentContainer.sharedLock.lock(); 613 try 614 { 615 currentContainer.renameEntry(currentDN, entry, modifyDNOperation); 616 } 617 catch (DatabaseException e) 618 { 619 logger.traceException(e); 620 throw createDirectoryException(e); 621 } 622 finally 623 { 624 currentContainer.sharedLock.unlock(); 625 writerEnd(); 626 } 627 } 628 629 630 631 /** {@inheritDoc} */ 632 @Override 633 public void search(SearchOperation searchOperation) 634 throws DirectoryException, CanceledOperationException 635 { 636 readerBegin(); 637 638 checkRootContainerInitialized(); 639 EntryContainer ec = rootContainer.getEntryContainer(searchOperation.getBaseDN()); 640 ec.sharedLock.lock(); 641 642 try 643 { 644 ec.search(searchOperation); 645 } 646 catch (DatabaseException e) 647 { 648 logger.traceException(e); 649 throw createDirectoryException(e); 650 } 651 finally 652 { 653 ec.sharedLock.unlock(); 654 readerEnd(); 655 } 656 } 657 658 private void checkRootContainerInitialized() throws DirectoryException 659 { 660 if (rootContainer == null) 661 { 662 LocalizableMessage msg = ERR_ROOT_CONTAINER_NOT_INITIALIZED.get(getBackendID()); 663 throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), msg); 664 } 665 } 666 667 /** {@inheritDoc} */ 668 @Override 669 public void exportLDIF(LDIFExportConfig exportConfig) 670 throws DirectoryException 671 { 672 // If the backend already has the root container open, we must use the same 673 // underlying root container 674 boolean openRootContainer = mustOpenRootContainer(); 675 final ResultCode errorRC = DirectoryServer.getServerErrorResultCode(); 676 try 677 { 678 if (openRootContainer) 679 { 680 rootContainer = getReadOnlyRootContainer(); 681 } 682 683 ExportJob exportJob = new ExportJob(exportConfig); 684 exportJob.exportLDIF(rootContainer); 685 } 686 catch (IOException ioe) 687 { 688 logger.traceException(ioe); 689 throw new DirectoryException(errorRC, ERR_EXPORT_IO_ERROR.get(ioe.getMessage()), ioe); 690 } 691 catch (DatabaseException de) 692 { 693 logger.traceException(de); 694 throw createDirectoryException(de); 695 } 696 catch (ConfigException ce) 697 { 698 throw new DirectoryException(errorRC, ce.getMessageObject(), ce); 699 } 700 catch (IdentifiedException e) 701 { 702 if (e instanceof DirectoryException) 703 { 704 throw (DirectoryException) e; 705 } 706 logger.traceException(e); 707 throw new DirectoryException(errorRC, e.getMessageObject(), e); 708 } 709 finally 710 { 711 closeTemporaryRootContainer(openRootContainer); 712 } 713 } 714 715 private boolean mustOpenRootContainer() 716 { 717 return rootContainer == null; 718 } 719 720 /** {@inheritDoc} */ 721 @Override 722 public LDIFImportResult importLDIF(LDIFImportConfig importConfig, ServerContext serverContext) 723 throws DirectoryException 724 { 725 RuntimeInformation.logInfo(); 726 727 // If the backend already has the root container open, we must use the same 728 // underlying root container 729 boolean openRootContainer = rootContainer == null; 730 731 // If the rootContainer is open, the backend is initialized by something else. 732 // We can't do import while the backend is online. 733 final ResultCode errorRC = DirectoryServer.getServerErrorResultCode(); 734 if(!openRootContainer) 735 { 736 throw new DirectoryException(errorRC, ERR_IMPORT_BACKEND_ONLINE.get()); 737 } 738 739 try 740 { 741 if (Importer.mustClearBackend(importConfig, cfg)) 742 { 743 // We have the writer lock on the environment, now delete the 744 // environment and re-open it. Only do this when we are 745 // importing to all the base DNs in the backend or if the backend only 746 // have one base DN. 747 File parentDirectory = getFileForPath(cfg.getDBDirectory()); 748 File backendDirectory = new File(parentDirectory, cfg.getBackendId()); 749 // If the backend does not exist the import will create it. 750 if (backendDirectory.exists()) 751 { 752 EnvManager.removeFiles(backendDirectory.getPath()); 753 } 754 } 755 756 final EnvironmentConfig envConfig = getEnvConfigForImport(); 757 final Importer importer = new Importer(importConfig, cfg, envConfig, serverContext); 758 rootContainer = initializeRootContainer(envConfig); 759 return importer.processImport(rootContainer); 760 } 761 catch (ExecutionException execEx) 762 { 763 logger.traceException(execEx); 764 if (execEx.getCause() instanceof DirectoryException) 765 { 766 throw ((DirectoryException) execEx.getCause()); 767 } 768 throw new DirectoryException(errorRC, ERR_EXECUTION_ERROR.get(execEx.getMessage())); 769 } 770 catch (InterruptedException intEx) 771 { 772 logger.traceException(intEx); 773 throw new DirectoryException(errorRC, ERR_INTERRUPTED_ERROR.get(intEx.getMessage())); 774 } 775 catch (JebException | InitializationException | ConfigException e) 776 { 777 logger.traceException(e); 778 throw new DirectoryException(errorRC, e.getMessageObject()); 779 } 780 finally 781 { 782 // leave the backend in the same state. 783 try 784 { 785 if (rootContainer != null) 786 { 787 long startTime = System.currentTimeMillis(); 788 rootContainer.close(); 789 long finishTime = System.currentTimeMillis(); 790 long closeTime = (finishTime - startTime) / 1000; 791 logger.info(NOTE_IMPORT_LDIF_ROOTCONTAINER_CLOSE, closeTime); 792 rootContainer = null; 793 } 794 795 // Sync the environment to disk. 796 logger.info(NOTE_IMPORT_CLOSING_DATABASE); 797 } 798 catch (DatabaseException de) 799 { 800 logger.traceException(de); 801 } 802 } 803 } 804 805 private EnvironmentConfig getEnvConfigForImport() 806 { 807 final EnvironmentConfig envConfig = new EnvironmentConfig(); 808 envConfig.setAllowCreate(true); 809 envConfig.setTransactional(false); 810 envConfig.setDurability(Durability.COMMIT_NO_SYNC); 811 envConfig.setLockTimeout(0, TimeUnit.SECONDS); 812 envConfig.setTxnTimeout(0, TimeUnit.SECONDS); 813 envConfig.setConfigParam(CLEANER_MIN_FILE_UTILIZATION, 814 String.valueOf(cfg.getDBCleanerMinUtilization())); 815 envConfig.setConfigParam(LOG_FILE_MAX, 816 String.valueOf(cfg.getDBLogFileMax())); 817 return envConfig; 818 } 819 820 /** {@inheritDoc} */ 821 @Override 822 public long verifyBackend(VerifyConfig verifyConfig) 823 throws InitializationException, ConfigException, DirectoryException 824 { 825 // If the backend already has the root container open, we must use the same 826 // underlying root container 827 final boolean openRootContainer = mustOpenRootContainer(); 828 try 829 { 830 if (openRootContainer) 831 { 832 rootContainer = getReadOnlyRootContainer(); 833 } 834 835 VerifyJob verifyJob = new VerifyJob(verifyConfig); 836 return verifyJob.verifyBackend(rootContainer); 837 } 838 catch (DatabaseException e) 839 { 840 logger.traceException(e); 841 throw createDirectoryException(e); 842 } 843 catch (JebException e) 844 { 845 logger.traceException(e); 846 throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), 847 e.getMessageObject()); 848 } 849 finally 850 { 851 closeTemporaryRootContainer(openRootContainer); 852 } 853 } 854 855 856 /** {@inheritDoc} */ 857 @Override 858 public void rebuildBackend(RebuildConfig rebuildConfig, ServerContext serverContext) 859 throws InitializationException, ConfigException, DirectoryException 860 { 861 // If the backend already has the root container open, we must use the same 862 // underlying root container 863 boolean openRootContainer = mustOpenRootContainer(); 864 865 /* 866 * If the rootContainer is open, the backend is initialized by something 867 * else. We can't do any rebuild of system indexes while others are using 868 * this backend. 869 */ 870 final ResultCode errorRC = DirectoryServer.getServerErrorResultCode(); 871 if(!openRootContainer && rebuildConfig.includesSystemIndex()) 872 { 873 throw new DirectoryException(errorRC, ERR_REBUILD_BACKEND_ONLINE.get()); 874 } 875 876 try 877 { 878 final EnvironmentConfig envConfig; 879 if (openRootContainer) 880 { 881 envConfig = getEnvConfigForImport(); 882 rootContainer = initializeRootContainer(envConfig); 883 } 884 else 885 { 886 envConfig = parseConfigEntry(cfg); 887 888 } 889 final Importer importer = new Importer(rebuildConfig, cfg, envConfig, serverContext); 890 importer.rebuildIndexes(rootContainer); 891 } 892 catch (ExecutionException execEx) 893 { 894 logger.traceException(execEx); 895 throw new DirectoryException(errorRC, ERR_EXECUTION_ERROR.get(execEx.getMessage())); 896 } 897 catch (InterruptedException intEx) 898 { 899 logger.traceException(intEx); 900 throw new DirectoryException(errorRC, ERR_INTERRUPTED_ERROR.get(intEx.getMessage())); 901 } 902 catch (ConfigException | JebException e) 903 { 904 logger.traceException(e); 905 throw new DirectoryException(errorRC, e.getMessageObject()); 906 } 907 catch (InitializationException e) 908 { 909 logger.traceException(e); 910 throw new InitializationException(e.getMessageObject()); 911 } 912 finally 913 { 914 closeTemporaryRootContainer(openRootContainer); 915 } 916 } 917 918 /** 919 * If a root container was opened in the calling method method as read only, 920 * close it to leave the backend in the same state. 921 */ 922 private void closeTemporaryRootContainer(boolean openRootContainer) 923 { 924 if (openRootContainer && rootContainer != null) 925 { 926 try 927 { 928 rootContainer.close(); 929 rootContainer = null; 930 } 931 catch (DatabaseException e) 932 { 933 logger.traceException(e); 934 } 935 } 936 } 937 938 939 /** {@inheritDoc} */ 940 @Override 941 public void createBackup(BackupConfig backupConfig) throws DirectoryException 942 { 943 new BackupManager(getBackendID()).createBackup(this, backupConfig); 944 } 945 946 /** {@inheritDoc} */ 947 @Override 948 public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException 949 { 950 new BackupManager(getBackendID()).removeBackup(backupDirectory, backupID); 951 } 952 953 /** {@inheritDoc} */ 954 @Override 955 public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException 956 { 957 new BackupManager(getBackendID()).restoreBackup(this, restoreConfig); 958 } 959 960 /** {@inheritDoc} */ 961 @Override 962 public ListIterator<Path> getFilesToBackup() throws DirectoryException 963 { 964 return new JELogFilesIterator(getDirectory(), cfg.getBackendId()); 965 } 966 967 /** 968 * Iterator on JE log files to backup. 969 * <p> 970 * The cleaner thread may delete some log files during the backup. The 971 * iterator is automatically renewed if at least one file has been deleted. 972 */ 973 static class JELogFilesIterator implements ListIterator<Path> 974 { 975 /** Underlying iterator on files. */ 976 private ListIterator<Path> iterator; 977 978 /** Root directory where all files are located. */ 979 private final File rootDirectory; 980 981 private final String backendID; 982 983 /** Files to backup. Used to renew the iterator if necessary. */ 984 private List<Path> files; 985 986 private String lastFileName = ""; 987 private long lastFileSize; 988 989 JELogFilesIterator(File rootDirectory, String backendID) throws DirectoryException 990 { 991 this.rootDirectory = rootDirectory; 992 this.backendID = backendID; 993 setFiles(BackupManager.getFiles(rootDirectory, new JELogFileFilter(), backendID)); 994 } 995 996 private void setFiles(List<Path> files) { 997 this.files = files; 998 Collections.sort(files); 999 if (!files.isEmpty()) 1000 { 1001 Path lastFile = files.get(files.size() - 1); 1002 lastFileName = lastFile.getFileName().toString(); 1003 lastFileSize = lastFile.toFile().length(); 1004 } 1005 iterator = files.listIterator(); 1006 } 1007 1008 /** {@inheritDoc} */ 1009 @Override 1010 public boolean hasNext() 1011 { 1012 boolean hasNext = iterator.hasNext(); 1013 if (!hasNext && !files.isEmpty()) 1014 { 1015 try 1016 { 1017 List<Path> allFiles = BackupManager.getFiles(rootDirectory, new JELogFileFilter(), backendID); 1018 List<Path> compare = new ArrayList<>(files); 1019 compare.removeAll(allFiles); 1020 if (!compare.isEmpty()) 1021 { 1022 // at least one file was deleted, the iterator must be renewed based on last file previously available 1023 List<Path> newFiles = 1024 BackupManager.getFiles(rootDirectory, new JELogFileFilter(lastFileName, lastFileSize), backendID); 1025 logger.info(NOTE_JEB_BACKUP_CLEANER_ACTIVITY.get(newFiles.size())); 1026 if (!newFiles.isEmpty()) 1027 { 1028 setFiles(newFiles); 1029 hasNext = iterator.hasNext(); 1030 } 1031 } 1032 } 1033 catch (DirectoryException e) 1034 { 1035 logger.error(ERR_BACKEND_LIST_FILES_TO_BACKUP.get(backendID, stackTraceToSingleLineString(e))); 1036 } 1037 } 1038 return hasNext; 1039 } 1040 1041 /** {@inheritDoc} */ 1042 @Override 1043 public Path next() 1044 { 1045 if (hasNext()) { 1046 return iterator.next(); 1047 } 1048 throw new NoSuchElementException(); 1049 } 1050 1051 /** {@inheritDoc} */ 1052 @Override 1053 public boolean hasPrevious() 1054 { 1055 return iterator.hasPrevious(); 1056 } 1057 1058 /** {@inheritDoc} */ 1059 @Override 1060 public Path previous() 1061 { 1062 return iterator.previous(); 1063 } 1064 1065 /** {@inheritDoc} */ 1066 @Override 1067 public int nextIndex() 1068 { 1069 return iterator.nextIndex(); 1070 } 1071 1072 /** {@inheritDoc} */ 1073 @Override 1074 public int previousIndex() 1075 { 1076 return iterator.previousIndex(); 1077 } 1078 1079 /** {@inheritDoc} */ 1080 @Override 1081 public void remove() 1082 { 1083 throw new UnsupportedOperationException("remove() is not implemented"); 1084 } 1085 1086 /** {@inheritDoc} */ 1087 @Override 1088 public void set(Path e) 1089 { 1090 throw new UnsupportedOperationException("set() is not implemented"); 1091 } 1092 1093 /** {@inheritDoc} */ 1094 @Override 1095 public void add(Path e) 1096 { 1097 throw new UnsupportedOperationException("add() is not implemented"); 1098 } 1099 1100 } 1101 1102 /** 1103 * This class implements a FilenameFilter to detect a JE log file, possibly with a constraint 1104 * on the file name and file size. 1105 */ 1106 private static class JELogFileFilter implements FileFilter { 1107 1108 private final String latestFilename; 1109 private final long latestFileSize; 1110 1111 /** 1112 * Creates the filter for log files that are newer than provided file name 1113 * or equal to provided file name and of larger size. 1114 */ 1115 JELogFileFilter(String latestFilename, long latestFileSize) { 1116 this.latestFilename = latestFilename; 1117 this.latestFileSize = latestFileSize; 1118 } 1119 1120 /** Creates the filter for any JE log file. */ 1121 JELogFileFilter() { 1122 this("", 0); 1123 } 1124 1125 /** {@inheritDoc} */ 1126 @Override 1127 public boolean accept(File file) 1128 { 1129 String name = file.getName(); 1130 int cmp = name.compareTo(latestFilename); 1131 return name.endsWith(".jdb") && (cmp > 0 || (cmp == 0 && file.length() > latestFileSize)); 1132 } 1133 } 1134 1135 /** {@inheritDoc} */ 1136 @Override 1137 public boolean isDirectRestore() 1138 { 1139 // restore is done in an intermediate directory 1140 return false; 1141 } 1142 1143 /** {@inheritDoc} */ 1144 @Override 1145 public Path beforeRestore() throws DirectoryException 1146 { 1147 return null; 1148 } 1149 1150 /** {@inheritDoc} */ 1151 @Override 1152 public void afterRestore(Path restoreDirectory, Path saveDirectory) throws DirectoryException 1153 { 1154 // intermediate directory content is moved to database directory 1155 File targetDirectory = getDirectory(); 1156 recursiveDelete(targetDirectory); 1157 try 1158 { 1159 Files.move(restoreDirectory, targetDirectory.toPath()); 1160 } 1161 catch(IOException e) 1162 { 1163 LocalizableMessage msg = ERR_CANNOT_RENAME_RESTORE_DIRECTORY.get(restoreDirectory, targetDirectory.getPath()); 1164 throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), msg); 1165 } 1166 } 1167 1168 /** {@inheritDoc} */ 1169 @Override 1170 public boolean isConfigurationAcceptable(LocalDBBackendCfg config, 1171 List<LocalizableMessage> unacceptableReasons, 1172 ServerContext serverContext) 1173 { 1174 return isConfigurationChangeAcceptable(config, unacceptableReasons); 1175 } 1176 1177 1178 1179 /** {@inheritDoc} */ 1180 @Override 1181 public boolean isConfigurationChangeAcceptable( 1182 LocalDBBackendCfg cfg, 1183 List<LocalizableMessage> unacceptableReasons) 1184 { 1185 // Make sure that the logging level value is acceptable. 1186 try { 1187 Level.parse(cfg.getDBLoggingLevel()); 1188 return true; 1189 } catch (Exception e) { 1190 unacceptableReasons.add(ERR_JEB_INVALID_LOGGING_LEVEL.get(cfg.getDBLoggingLevel(), cfg.dn())); 1191 return false; 1192 } 1193 } 1194 1195 1196 1197 /** {@inheritDoc} */ 1198 @Override 1199 public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg newCfg) 1200 { 1201 final ConfigChangeResult ccr = new ConfigChangeResult(); 1202 try 1203 { 1204 if(rootContainer != null) 1205 { 1206 SortedSet<DN> newBaseDNs = newCfg.getBaseDN(); 1207 DN[] newBaseDNsArray = newBaseDNs.toArray(new DN[newBaseDNs.size()]); 1208 1209 // Check for changes to the base DNs. 1210 removeDeletedBaseDNs(newBaseDNs); 1211 ConfigChangeResult failure = createNewBaseDNs(newBaseDNsArray, ccr); 1212 if (failure != null) 1213 { 1214 return failure; 1215 } 1216 1217 baseDNs = newBaseDNsArray; 1218 } 1219 1220 updateDiskMonitor(diskMonitor, newCfg); 1221 1222 // Put the new configuration in place. 1223 this.cfg = newCfg; 1224 } 1225 catch (Exception e) 1226 { 1227 ccr.addMessage(LocalizableMessage.raw(stackTraceToSingleLineString(e))); 1228 ccr.setResultCode(DirectoryServer.getServerErrorResultCode()); 1229 } 1230 return ccr; 1231 } 1232 1233 private void updateDiskMonitor(DiskSpaceMonitor dm, LocalDBBackendCfg newCfg) 1234 { 1235 diskMonitor.registerMonitoredDirectory(getBackendID(), getDirectory(), newCfg.getDiskLowThreshold(), 1236 newCfg.getDiskFullThreshold(), this); 1237 } 1238 1239 private void removeDeletedBaseDNs(SortedSet<DN> newBaseDNs) throws DirectoryException 1240 { 1241 for (DN baseDN : cfg.getBaseDN()) 1242 { 1243 if (!newBaseDNs.contains(baseDN)) 1244 { 1245 // The base DN was deleted. 1246 DirectoryServer.deregisterBaseDN(baseDN); 1247 EntryContainer ec = rootContainer.unregisterEntryContainer(baseDN); 1248 ec.close(); 1249 ec.delete(); 1250 } 1251 } 1252 } 1253 1254 private ConfigChangeResult createNewBaseDNs(DN[] newBaseDNsArray, final ConfigChangeResult ccr) 1255 { 1256 for (DN baseDN : newBaseDNsArray) 1257 { 1258 if (!rootContainer.getBaseDNs().contains(baseDN)) 1259 { 1260 try 1261 { 1262 // The base DN was added. 1263 EntryContainer ec = rootContainer.openEntryContainer(baseDN, null); 1264 rootContainer.registerEntryContainer(baseDN, ec); 1265 DirectoryServer.registerBaseDN(baseDN, this, false); 1266 } 1267 catch (Exception e) 1268 { 1269 logger.traceException(e); 1270 1271 ccr.setResultCode(DirectoryServer.getServerErrorResultCode()); 1272 ccr.addMessage(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(baseDN, e)); 1273 return ccr; 1274 } 1275 } 1276 } 1277 return null; 1278 } 1279 1280 /** 1281 * Returns a handle to the JE root container currently used by this backend. 1282 * The rootContainer could be NULL if the backend is not initialized. 1283 * 1284 * @return The RootContainer object currently used by this backend. 1285 */ 1286 public RootContainer getRootContainer() 1287 { 1288 return rootContainer; 1289 } 1290 1291 /** 1292 * Returns a new read-only handle to the JE root container for this backend. 1293 * The caller is responsible for closing the root container after use. 1294 * 1295 * @return The read-only RootContainer object for this backend. 1296 * 1297 * @throws ConfigException If an unrecoverable problem arises during 1298 * initialization. 1299 * @throws InitializationException If a problem occurs during initialization 1300 * that is not related to the server 1301 * configuration. 1302 */ 1303 public RootContainer getReadOnlyRootContainer() 1304 throws ConfigException, InitializationException 1305 { 1306 EnvironmentConfig envConfig = parseConfigEntry(cfg); 1307 1308 envConfig.setReadOnly(true); 1309 envConfig.setAllowCreate(false); 1310 envConfig.setTransactional(false); 1311 envConfig.setConfigParam(ENV_IS_LOCKING, "true"); 1312 envConfig.setConfigParam(ENV_RUN_CHECKPOINTER, "true"); 1313 1314 return initializeRootContainer(envConfig); 1315 } 1316 1317 /** 1318 * Clears all the entries from the backend. This method is for test cases 1319 * that use the JE backend. 1320 * 1321 * @throws ConfigException If an unrecoverable problem arises in the 1322 * process of performing the initialization. 1323 * 1324 * @throws JebException If an error occurs while removing the data. 1325 */ 1326 public void clearBackend() 1327 throws ConfigException, JebException 1328 { 1329 // Determine the backend database directory. 1330 File parentDirectory = getFileForPath(cfg.getDBDirectory()); 1331 File backendDirectory = new File(parentDirectory, cfg.getBackendId()); 1332 EnvManager.removeFiles(backendDirectory.getPath()); 1333 } 1334 1335 /** 1336 * Creates a customized DirectoryException from the DatabaseException thrown 1337 * by JE backend. 1338 * 1339 * @param e The DatabaseException to be converted. 1340 * @return DirectoryException created from exception. 1341 */ 1342 private DirectoryException createDirectoryException(DatabaseException e) { 1343 if (e instanceof EnvironmentFailureException && !rootContainer.isValid()) { 1344 LocalizableMessage message = NOTE_BACKEND_ENVIRONMENT_UNUSABLE.get(getBackendID()); 1345 logger.info(message); 1346 DirectoryServer.sendAlertNotification(DirectoryServer.getInstance(), 1347 ALERT_TYPE_BACKEND_ENVIRONMENT_UNUSABLE, message); 1348 } 1349 1350 String jeMessage = e.getMessage(); 1351 if (jeMessage == null) { 1352 jeMessage = stackTraceToSingleLineString(e); 1353 } 1354 LocalizableMessage message = ERR_DATABASE_EXCEPTION.get(jeMessage); 1355 return new DirectoryException( 1356 DirectoryServer.getServerErrorResultCode(), message, e); 1357 } 1358 1359 /** {@inheritDoc} */ 1360 @Override 1361 public String getClassName() { 1362 return BackendImpl.class.getName(); 1363 } 1364 1365 /** {@inheritDoc} */ 1366 @Override 1367 public Map<String, String> getAlerts() 1368 { 1369 Map<String, String> alerts = new LinkedHashMap<>(); 1370 1371 alerts.put(ALERT_TYPE_BACKEND_ENVIRONMENT_UNUSABLE, 1372 ALERT_DESCRIPTION_BACKEND_ENVIRONMENT_UNUSABLE); 1373 alerts.put(ALERT_TYPE_DISK_SPACE_LOW, 1374 ALERT_DESCRIPTION_DISK_SPACE_LOW); 1375 alerts.put(ALERT_TYPE_DISK_FULL, 1376 ALERT_DESCRIPTION_DISK_FULL); 1377 return alerts; 1378 } 1379 1380 /** {@inheritDoc} */ 1381 @Override 1382 public DN getComponentEntryDN() { 1383 return cfg.dn(); 1384 } 1385 1386 private RootContainer initializeRootContainer(EnvironmentConfig envConfig) 1387 throws ConfigException, InitializationException { 1388 // Open the database environment 1389 try { 1390 RootContainer rc = new RootContainer(this, cfg); 1391 rc.open(envConfig); 1392 return rc; 1393 } 1394 catch (DatabaseException e) { 1395 logger.traceException(e); 1396 LocalizableMessage message = ERR_OPEN_ENV_FAIL.get(e.getMessage()); 1397 throw new InitializationException(message, e); 1398 } 1399 } 1400 1401 /** {@inheritDoc} */ 1402 @Override 1403 public void diskLowThresholdReached(File directory, long thresholdInBytes) { 1404 storageStatus = StorageStatus.lockedDown( 1405 WARN_DISK_SPACE_LOW_THRESHOLD_CROSSED.get(directory.getFreeSpace(), directory.getAbsolutePath(), 1406 thresholdInBytes, getBackendID())); 1407 } 1408 1409 /** {@inheritDoc} */ 1410 @Override 1411 public void diskFullThresholdReached(File directory, long thresholdInBytes) { 1412 storageStatus = StorageStatus.unusable( 1413 WARN_DISK_SPACE_FULL_THRESHOLD_CROSSED.get(directory.getFreeSpace(), directory.getAbsolutePath(), 1414 thresholdInBytes, getBackendID())); 1415 } 1416 1417 /** {@inheritDoc} */ 1418 @Override 1419 public void diskSpaceRestored(File directory, long lowThresholdInBytes, long fullThresholdInBytes) { 1420 storageStatus = StorageStatus.working(); 1421 } 1422 1423 private void checkDiskSpace(Operation operation) throws DirectoryException 1424 { 1425 if(storageStatus.isUnusable() || 1426 (storageStatus.isLockedDown() 1427 && operation != null 1428 && !operation.getClientConnection().hasPrivilege( 1429 Privilege.BYPASS_LOCKDOWN, operation))) 1430 { 1431 throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, 1432 WARN_OUT_OF_DISK_SPACE.get()); 1433 } 1434 } 1435}