001/*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License").  You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
010 * or http://forgerock.org/license/CDDLv1.0.html.
011 * See the License for the specific language governing permissions
012 * and limitations under the License.
013 *
014 * When distributing Covered Code, include this CDDL HEADER in each
015 * file and include the License file at legal-notices/CDDLv1_0.txt.
016 * If applicable, add the following below this CDDL HEADER, with the
017 * fields enclosed by brackets "[]" replaced with your own identifying
018 * information:
019 *      Portions Copyright [yyyy] [name of copyright owner]
020 *
021 * CDDL HEADER END
022 *
023 *
024 *      Copyright 2006-2010 Sun Microsystems, Inc.
025 *      Portions Copyright 2011-2015 ForgeRock AS
026 *      Portions copyright 2013 Manuel Gaupp
027 */
028package org.opends.server.backends.jeb;
029
030import java.util.*;
031import java.util.concurrent.locks.Lock;
032import java.util.concurrent.locks.ReentrantReadWriteLock;
033
034import org.forgerock.i18n.LocalizableMessage;
035import org.forgerock.i18n.LocalizableMessageBuilder;
036import org.forgerock.i18n.slf4j.LocalizedLogger;
037import org.forgerock.opendj.config.server.ConfigChangeResult;
038import org.forgerock.opendj.config.server.ConfigException;
039import org.forgerock.opendj.ldap.ByteString;
040import org.forgerock.opendj.ldap.ResultCode;
041import org.forgerock.opendj.ldap.SearchScope;
042import org.forgerock.util.Utils;
043import org.opends.server.admin.server.ConfigurationAddListener;
044import org.opends.server.admin.server.ConfigurationChangeListener;
045import org.opends.server.admin.server.ConfigurationDeleteListener;
046import org.opends.server.admin.std.server.LocalDBBackendCfg;
047import org.opends.server.admin.std.server.LocalDBIndexCfg;
048import org.opends.server.admin.std.server.LocalDBVLVIndexCfg;
049import org.opends.server.api.ClientConnection;
050import org.opends.server.api.EntryCache;
051import org.opends.server.api.plugin.PluginResult.SubordinateDelete;
052import org.opends.server.api.plugin.PluginResult.SubordinateModifyDN;
053import org.opends.server.backends.pluggable.SuffixContainer;
054import org.opends.server.controls.*;
055import org.opends.server.core.*;
056import org.opends.server.types.*;
057import org.opends.server.util.ServerConstants;
058import org.opends.server.util.StaticUtils;
059
060import com.sleepycat.je.*;
061
062import static com.sleepycat.je.LockMode.*;
063import static org.opends.messages.BackendMessages.*;
064import static org.opends.server.backends.jeb.JebFormat.*;
065import static org.opends.server.core.DirectoryServer.*;
066import static org.opends.server.protocols.ldap.LDAPResultCode.*;
067import static org.opends.server.types.AdditionalLogItem.*;
068import static org.opends.server.util.StaticUtils.*;
069
070/**
071 * Storage container for LDAP entries.  Each base DN of a JE backend is given
072 * its own entry container.  The entry container is the object that implements
073 * the guts of the backend API methods for LDAP operations.
074 */
075public class EntryContainer
076    implements SuffixContainer, ConfigurationChangeListener<LocalDBBackendCfg>
077{
078  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
079
080  /** The name of the entry database. */
081  public static final String ID2ENTRY_DATABASE_NAME = ID2ENTRY_INDEX_NAME;
082  /** The name of the DN database. */
083  public static final String DN2ID_DATABASE_NAME = DN2ID_INDEX_NAME;
084  /** The name of the children index database. */
085  private static final String ID2CHILDREN_DATABASE_NAME = ID2CHILDREN_INDEX_NAME;
086  /** The name of the subtree index database. */
087  private static final String ID2SUBTREE_DATABASE_NAME = ID2SUBTREE_INDEX_NAME;
088  /** The name of the referral database. */
089  private static final String REFERRAL_DATABASE_NAME = REFERRAL_INDEX_NAME;
090  /** The name of the state database. */
091  private static final String STATE_DATABASE_NAME = STATE_INDEX_NAME;
092
093  /** The attribute index configuration manager. */
094  private final AttributeJEIndexCfgManager attributeJEIndexCfgManager;
095  /** The vlv index configuration manager. */
096  private final VLVJEIndexCfgManager vlvJEIndexCfgManager;
097
098  /** ID of the backend to which this entry container belongs. */
099  private final String backendID;
100
101  /** The root container in which this entryContainer belongs. */
102  private final RootContainer rootContainer;
103
104  /** The baseDN this entry container is responsible for. */
105  private final DN baseDN;
106
107  /** The backend configuration. */
108  private LocalDBBackendCfg config;
109
110  /** The JE database environment. */
111  private final Environment env;
112
113  /** The DN database maps a normalized DN string to an entry ID (8 bytes). */
114  private DN2ID dn2id;
115  /** The entry database maps an entry ID (8 bytes) to a complete encoded entry. */
116  private ID2Entry id2entry;
117  /** Index maps entry ID to an entry ID list containing its children. */
118  private Index id2children;
119  /** Index maps entry ID to an entry ID list containing its subordinates. */
120  private Index id2subtree;
121  /** The referral database maps a normalized DN string to labeled URIs. */
122  private DN2URI dn2uri;
123  /** The state database maps a config DN to config entries. */
124  private State state;
125
126  /** The set of attribute indexes. */
127  private final HashMap<AttributeType, AttributeIndex> attrIndexMap = new HashMap<>();
128  /** The set of VLV (Virtual List View) indexes. */
129  private final HashMap<String, VLVIndex> vlvIndexMap = new HashMap<>();
130
131  /**
132   * Prevents name clashes for common indexes (like id2entry) across multiple suffixes.
133   * For example when a root container contains multiple suffixes.
134   */
135  private String databasePrefix;
136
137  /**
138   * This class is responsible for managing the configuration for attribute
139   * indexes used within this entry container.
140   */
141  private class AttributeJEIndexCfgManager implements
142  ConfigurationAddListener<LocalDBIndexCfg>,
143  ConfigurationDeleteListener<LocalDBIndexCfg>
144  {
145    /** {@inheritDoc} */
146    @Override
147    public boolean isConfigurationAddAcceptable(
148        LocalDBIndexCfg cfg,
149        List<LocalizableMessage> unacceptableReasons)
150    {
151      try
152      {
153        //Try creating all the indexes before confirming they are valid ones.
154        new AttributeIndex(cfg, EntryContainer.this);
155        return true;
156      }
157      catch(Exception e)
158      {
159        unacceptableReasons.add(LocalizableMessage.raw(e.getLocalizedMessage()));
160        return false;
161      }
162    }
163
164    /** {@inheritDoc} */
165    @Override
166    public ConfigChangeResult applyConfigurationAdd(LocalDBIndexCfg cfg)
167    {
168      final ConfigChangeResult ccr = new ConfigChangeResult();
169
170      try
171      {
172        AttributeIndex index = new AttributeIndex(cfg, EntryContainer.this);
173        index.open();
174        if(!index.isTrusted())
175        {
176          ccr.setAdminActionRequired(true);
177          ccr.addMessage(NOTE_INDEX_ADD_REQUIRES_REBUILD.get(cfg.getAttribute().getNameOrOID()));
178        }
179        attrIndexMap.put(cfg.getAttribute(), index);
180      }
181      catch(Exception e)
182      {
183        ccr.addMessage(LocalizableMessage.raw(e.getLocalizedMessage()));
184        ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
185      }
186
187      return ccr;
188    }
189
190    /** {@inheritDoc} */
191    @Override
192    public boolean isConfigurationDeleteAcceptable(
193        LocalDBIndexCfg cfg, List<LocalizableMessage> unacceptableReasons)
194    {
195      // TODO: validate more before returning true?
196      return true;
197    }
198
199    /** {@inheritDoc} */
200    @Override
201    public ConfigChangeResult applyConfigurationDelete(LocalDBIndexCfg cfg)
202    {
203      final ConfigChangeResult ccr = new ConfigChangeResult();
204
205      exclusiveLock.lock();
206      try
207      {
208        AttributeIndex index = attrIndexMap.get(cfg.getAttribute());
209        deleteAttributeIndex(index);
210        attrIndexMap.remove(cfg.getAttribute());
211      }
212      catch(DatabaseException de)
213      {
214        ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
215        ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
216      }
217      finally
218      {
219        exclusiveLock.unlock();
220      }
221
222      return ccr;
223    }
224  }
225
226  /**
227   * This class is responsible for managing the configuration for VLV indexes
228   * used within this entry container.
229   */
230  private class VLVJEIndexCfgManager implements
231  ConfigurationAddListener<LocalDBVLVIndexCfg>,
232  ConfigurationDeleteListener<LocalDBVLVIndexCfg>
233  {
234    /** {@inheritDoc} */
235    @Override
236    public boolean isConfigurationAddAcceptable(
237        LocalDBVLVIndexCfg cfg, List<LocalizableMessage> unacceptableReasons)
238    {
239      try
240      {
241        SearchFilter.createFilterFromString(cfg.getFilter());
242      }
243      catch(Exception e)
244      {
245        unacceptableReasons.add(
246            ERR_CONFIG_VLV_INDEX_BAD_FILTER.get(cfg.getFilter(), cfg.getName(), e.getLocalizedMessage()));
247        return false;
248      }
249
250      String[] sortAttrs = cfg.getSortOrder().split(" ");
251      SortKey[] sortKeys = new SortKey[sortAttrs.length];
252      boolean[] ascending = new boolean[sortAttrs.length];
253      for(int i = 0; i < sortAttrs.length; i++)
254      {
255        try
256        {
257          if(sortAttrs[i].startsWith("-"))
258          {
259            ascending[i] = false;
260            sortAttrs[i] = sortAttrs[i].substring(1);
261          }
262          else
263          {
264            ascending[i] = true;
265            if(sortAttrs[i].startsWith("+"))
266            {
267              sortAttrs[i] = sortAttrs[i].substring(1);
268            }
269          }
270        }
271        catch(Exception e)
272        {
273          unacceptableReasons.add(ERR_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], cfg.getName()));
274          return false;
275        }
276
277        AttributeType attrType =
278          DirectoryServer.getAttributeType(sortAttrs[i].toLowerCase());
279        if(attrType == null)
280        {
281          unacceptableReasons.add(ERR_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortAttrs[i], cfg.getName()));
282          return false;
283        }
284        sortKeys[i] = new SortKey(attrType, ascending[i]);
285      }
286
287      return true;
288    }
289
290    /** {@inheritDoc} */
291    @Override
292    public ConfigChangeResult applyConfigurationAdd(LocalDBVLVIndexCfg cfg)
293    {
294      final ConfigChangeResult ccr = new ConfigChangeResult();
295
296      try
297      {
298        VLVIndex vlvIndex = new VLVIndex(cfg, state, env, EntryContainer.this);
299        vlvIndex.open();
300        if(!vlvIndex.isTrusted())
301        {
302          ccr.setAdminActionRequired(true);
303          ccr.addMessage(NOTE_INDEX_ADD_REQUIRES_REBUILD.get(cfg.getName()));
304        }
305        vlvIndexMap.put(cfg.getName().toLowerCase(), vlvIndex);
306      }
307      catch(Exception e)
308      {
309        ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(e)));
310        ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
311      }
312
313      return ccr;
314    }
315
316    /** {@inheritDoc} */
317    @Override
318    public boolean isConfigurationDeleteAcceptable(
319        LocalDBVLVIndexCfg cfg,
320        List<LocalizableMessage> unacceptableReasons)
321    {
322      // TODO: validate more before returning true?
323      return true;
324    }
325
326    /** {@inheritDoc} */
327    @Override
328    public ConfigChangeResult applyConfigurationDelete(LocalDBVLVIndexCfg cfg)
329    {
330      final ConfigChangeResult ccr = new ConfigChangeResult();
331
332      exclusiveLock.lock();
333      try
334      {
335        VLVIndex vlvIndex =
336          vlvIndexMap.get(cfg.getName().toLowerCase());
337        deleteDatabase(vlvIndex);
338        vlvIndexMap.remove(cfg.getName());
339      }
340      catch(DatabaseException de)
341      {
342        ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
343        ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
344      }
345      finally
346      {
347        exclusiveLock.unlock();
348      }
349
350      return ccr;
351    }
352
353  }
354
355  /** A read write lock to handle schema changes and bulk changes. */
356  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
357  final Lock sharedLock = lock.readLock();
358  final Lock exclusiveLock = lock.writeLock();
359
360  /**
361   * Create a new entry container object.
362   *
363   * @param baseDN  The baseDN this entry container will be responsible for
364   *                storing on disk.
365   * @param databasePrefix The prefix to use in the database names used by
366   *                       this entry container.
367   * @param backendID ID of the JE backend that is creating this entry
368   *                  container. It is needed by the Directory Server
369   *                  entry cache methods.
370   * @param config The configuration of the JE backend.
371   * @param env The JE environment to create this entryContainer in.
372   * @param rootContainer The root container this entry container is in.
373   * @throws ConfigException if a configuration related error occurs.
374   */
375  EntryContainer(DN baseDN, String databasePrefix, String backendID,
376      LocalDBBackendCfg config, Environment env, RootContainer rootContainer)
377          throws ConfigException
378  {
379    this.backendID = backendID;
380    this.baseDN = baseDN;
381    this.config = config;
382    this.env = env;
383    this.rootContainer = rootContainer;
384
385    this.databasePrefix = preparePrefix(databasePrefix);
386
387    config.addLocalDBChangeListener(this);
388
389    attributeJEIndexCfgManager = new AttributeJEIndexCfgManager();
390    config.addLocalDBIndexAddListener(attributeJEIndexCfgManager);
391    config.addLocalDBIndexDeleteListener(attributeJEIndexCfgManager);
392
393    vlvJEIndexCfgManager = new VLVJEIndexCfgManager();
394    config.addLocalDBVLVIndexAddListener(vlvJEIndexCfgManager);
395    config.addLocalDBVLVIndexDeleteListener(vlvJEIndexCfgManager);
396  }
397
398  /**
399   * Opens the entryContainer for reading and writing.
400   *
401   * @throws DatabaseException If an error occurs in the JE database.
402   * @throws ConfigException if a configuration related error occurs.
403   */
404  void open() throws DatabaseException, ConfigException
405  {
406    try
407    {
408      DataConfig entryDataConfig =
409        new DataConfig(config.isEntriesCompressed(),
410            config.isCompactEncoding(),
411            rootContainer.getCompressedSchema());
412
413      id2entry = new ID2Entry(databasePrefix + "_" + ID2ENTRY_DATABASE_NAME,
414          entryDataConfig, env, this);
415      id2entry.open();
416
417      dn2id = new DN2ID(databasePrefix + "_" + DN2ID_DATABASE_NAME, env, this);
418      dn2id.open();
419
420      state = new State(databasePrefix + "_" + STATE_DATABASE_NAME, env, this);
421      state.open();
422
423      if (config.isSubordinateIndexesEnabled())
424      {
425        openSubordinateIndexes();
426      }
427      else
428      {
429        // Use a null index and ensure that future attempts to use the real
430        // subordinate indexes will fail.
431        id2children = new NullIndex(databasePrefix + "_"
432            + ID2CHILDREN_DATABASE_NAME, new ID2CIndexer(), state, env, this);
433        if (!env.getConfig().getReadOnly())
434        {
435          state.putIndexTrustState(null, id2children, false);
436        }
437        id2children.open(); // No-op
438
439        id2subtree = new NullIndex(databasePrefix + "_"
440            + ID2SUBTREE_DATABASE_NAME, new ID2SIndexer(), state, env, this);
441        if (!env.getConfig().getReadOnly())
442        {
443          state.putIndexTrustState(null, id2subtree, false);
444        }
445        id2subtree.open(); // No-op
446
447        logger.info(NOTE_JEB_SUBORDINATE_INDEXES_DISABLED, backendID);
448      }
449
450      dn2uri = new DN2URI(databasePrefix + "_" + REFERRAL_DATABASE_NAME, env, this);
451      dn2uri.open();
452
453      for (String idx : config.listLocalDBIndexes())
454      {
455        LocalDBIndexCfg indexCfg = config.getLocalDBIndex(idx);
456
457        AttributeIndex index = new AttributeIndex(indexCfg, this);
458        index.open();
459        if(!index.isTrusted())
460        {
461          logger.info(NOTE_INDEX_ADD_REQUIRES_REBUILD, index.getName());
462        }
463        attrIndexMap.put(indexCfg.getAttribute(), index);
464      }
465
466      for(String idx : config.listLocalDBVLVIndexes())
467      {
468        LocalDBVLVIndexCfg vlvIndexCfg = config.getLocalDBVLVIndex(idx);
469
470        VLVIndex vlvIndex = new VLVIndex(vlvIndexCfg, state, env, this);
471        vlvIndex.open();
472
473        if(!vlvIndex.isTrusted())
474        {
475          logger.info(NOTE_INDEX_ADD_REQUIRES_REBUILD, vlvIndex.getName());
476        }
477
478        vlvIndexMap.put(vlvIndexCfg.getName().toLowerCase(), vlvIndex);
479      }
480    }
481    catch (DatabaseException de)
482    {
483      logger.traceException(de);
484      close();
485      throw de;
486    }
487  }
488
489  /**
490   * Closes the entry container.
491   *
492   * @throws DatabaseException If an error occurs in the JE database.
493   */
494  @Override
495  public void close() throws DatabaseException
496  {
497    // Close core indexes.
498    dn2id.close();
499    id2entry.close();
500    dn2uri.close();
501    id2children.close();
502    id2subtree.close();
503    state.close();
504
505    Utils.closeSilently(attrIndexMap.values());
506
507    for (VLVIndex vlvIndex : vlvIndexMap.values())
508    {
509      vlvIndex.close();
510    }
511
512    // Deregister any listeners.
513    config.removeLocalDBChangeListener(this);
514    config.removeLocalDBIndexAddListener(attributeJEIndexCfgManager);
515    config.removeLocalDBIndexDeleteListener(attributeJEIndexCfgManager);
516    config.removeLocalDBVLVIndexAddListener(vlvJEIndexCfgManager);
517    config.removeLocalDBVLVIndexDeleteListener(vlvJEIndexCfgManager);
518  }
519
520  /**
521   * Retrieves a reference to the root container in which this entry container
522   * exists.
523   *
524   * @return  A reference to the root container in which this entry container
525   *          exists.
526   */
527  public RootContainer getRootContainer()
528  {
529    return rootContainer;
530  }
531
532  /**
533   * Get the DN database used by this entry container.
534   * The entryContainer must have been opened.
535   *
536   * @return The DN database.
537   */
538  public DN2ID getDN2ID()
539  {
540    return dn2id;
541  }
542
543  /**
544   * Get the entry database used by this entry container.
545   * The entryContainer must have been opened.
546   *
547   * @return The entry database.
548   */
549  public ID2Entry getID2Entry()
550  {
551    return id2entry;
552  }
553
554  /**
555   * Get the referral database used by this entry container.
556   * The entryContainer must have been opened.
557   *
558   * @return The referral database.
559   */
560  public DN2URI getDN2URI()
561  {
562    return dn2uri;
563  }
564
565  /**
566   * Get the children database used by this entry container.
567   * The entryContainer must have been opened.
568   *
569   * @return The children database.
570   */
571  public Index getID2Children()
572  {
573    return id2children;
574  }
575
576  /**
577   * Get the subtree database used by this entry container.
578   * The entryContainer must have been opened.
579   *
580   * @return The subtree database.
581   */
582  public Index getID2Subtree()
583  {
584    return id2subtree;
585  }
586
587  /**
588   * Get the state database used by this entry container.
589   * The entry container must have been opened.
590   *
591   * @return The state database.
592   */
593  public State getState()
594  {
595    return state;
596  }
597
598  /**
599   * Look for an attribute index for the given attribute type.
600   *
601   * @param attrType The attribute type for which an attribute index is needed.
602   * @return The attribute index or null if there is none for that type.
603   */
604  AttributeIndex getAttributeIndex(AttributeType attrType)
605  {
606    return attrIndexMap.get(attrType);
607  }
608
609  /**
610   * Look for an VLV index for the given index name.
611   *
612   * @param vlvIndexName The vlv index name for which an vlv index is needed.
613   * @return The VLV index or null if there is none with that name.
614   */
615  VLVIndex getVLVIndex(String vlvIndexName)
616  {
617    return vlvIndexMap.get(vlvIndexName);
618  }
619
620  /**
621   * Retrieve all attribute indexes.
622   *
623   * @return All attribute indexes defined in this entry container.
624   */
625  public Collection<AttributeIndex> getAttributeIndexes()
626  {
627    return attrIndexMap.values();
628  }
629
630  /**
631   * Retrieve all VLV indexes.
632   *
633   * @return The collection of VLV indexes defined in this entry container.
634   */
635  public Collection<VLVIndex> getVLVIndexes()
636  {
637    return vlvIndexMap.values();
638  }
639
640  /**
641   * Determine the highest entryID in the entryContainer.
642   * The entryContainer must already be open.
643   *
644   * @return The highest entry ID.
645   * @throws DatabaseException If an error occurs in the JE database.
646   */
647  public EntryID getHighestEntryID() throws DatabaseException
648  {
649    Cursor cursor = id2entry.openCursor(null, null);
650    try
651    {
652      // Position a cursor on the last data item, and the key should give the highest ID.
653      DatabaseEntry key = new DatabaseEntry();
654      DatabaseEntry data = new DatabaseEntry();
655
656      if (cursor.getLast(key, data, DEFAULT) == OperationStatus.SUCCESS)
657      {
658        return new EntryID(key);
659      }
660      return new EntryID(0);
661    }
662    finally
663    {
664      cursor.close();
665    }
666  }
667
668  /**
669   * Determine the number of subordinate entries for a given entry.
670   *
671   * @param entryDN The distinguished name of the entry.
672   * @param subtree <code>true</code> will include the entry and all the
673   *                entries under the given entries. <code>false</code>
674   *                will only return the number of entries immediately
675   *                under the given entry.
676   * @return The number of subordinate entries for the given entry or -1 if
677   *         the entry does not exist.
678   * @throws DatabaseException If an error occurs in the JE database.
679   */
680  long getNumSubordinates(DN entryDN, boolean subtree)
681  throws DatabaseException
682  {
683    EntryID entryID = dn2id.get(null, entryDN, LockMode.DEFAULT);
684    if (entryID != null)
685    {
686      DatabaseEntry key = new DatabaseEntry(entryIDToDatabase(entryID.longValue()));
687      final EntryIDSet entryIDSet;
688      long count;
689      if (subtree)
690      {
691        count = dn2id.get(null, entryDN, LockMode.DEFAULT) != null ? 1 : 0;
692        entryIDSet = id2subtree.readKey(key, null, LockMode.DEFAULT);
693      }
694      else
695      {
696        count = 0;
697        entryIDSet = id2children.readKey(key, null, LockMode.DEFAULT);
698      }
699      if(entryIDSet.size() == Long.MAX_VALUE)
700      {
701        return -1;
702      }
703      return count + entryIDSet.size();
704    }
705    return -1;
706  }
707
708  /**
709   * Processes the specified search in this entryContainer.
710   * Matching entries should be provided back to the core server using the
711   * <CODE>SearchOperation.returnEntry</CODE> method.
712   *
713   * @param searchOperation The search operation to be processed.
714   * @throws DirectoryException
715   *          If a problem occurs while processing the search.
716   * @throws DatabaseException If an error occurs in the JE database.
717   * @throws CanceledOperationException if this operation should be cancelled.
718   */
719  void search(SearchOperation searchOperation)
720  throws DirectoryException, DatabaseException, CanceledOperationException
721  {
722    DN aBaseDN = searchOperation.getBaseDN();
723    SearchScope searchScope = searchOperation.getScope();
724
725    PagedResultsControl pageRequest = searchOperation
726    .getRequestControl(PagedResultsControl.DECODER);
727    ServerSideSortRequestControl sortRequest = searchOperation
728    .getRequestControl(ServerSideSortRequestControl.DECODER);
729    if(sortRequest != null && !sortRequest.containsSortKeys()
730            && sortRequest.isCritical())
731    {
732      /*
733         If the control's criticality field is true then the server SHOULD do
734         the following: return unavailableCriticalExtension as a return code
735         in the searchResultDone message; include the sortKeyResponseControl in
736         the searchResultDone message, and not send back any search result
737         entries.
738       */
739      searchOperation.addResponseControl(new ServerSideSortResponseControl(NO_SUCH_ATTRIBUTE, null));
740      searchOperation.setResultCode(ResultCode.UNAVAILABLE_CRITICAL_EXTENSION);
741      return;
742    }
743    VLVRequestControl vlvRequest = searchOperation.getRequestControl(VLVRequestControl.DECODER);
744
745    if (vlvRequest != null && pageRequest != null)
746    {
747      throw new DirectoryException(ResultCode.CONSTRAINT_VIOLATION, ERR_SEARCH_CANNOT_MIX_PAGEDRESULTS_AND_VLV.get());
748    }
749
750    // Handle client abandon of paged results.
751    if (pageRequest != null)
752    {
753      if (pageRequest.getSize() == 0)
754      {
755        Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
756        searchOperation.getResponseControls().add(control);
757        return;
758      }
759      if (searchOperation.getSizeLimit() > 0 &&
760        pageRequest.getSize() >= searchOperation.getSizeLimit())
761      {
762        // The RFC says : "If the page size is greater than or equal to the
763        // sizeLimit value, the server should ignore the control as the
764        // request can be satisfied in a single page"
765        pageRequest = null;
766      }
767    }
768
769    // Handle base-object search first.
770    if (searchScope == SearchScope.BASE_OBJECT)
771    {
772      // Fetch the base entry.
773      Entry baseEntry = fetchBaseEntry(aBaseDN, searchScope);
774
775      if (!isManageDsaITOperation(searchOperation))
776      {
777        dn2uri.checkTargetForReferral(baseEntry, searchOperation.getScope());
778      }
779
780      if (searchOperation.getFilter().matchesEntry(baseEntry))
781      {
782        searchOperation.returnEntry(baseEntry, null);
783      }
784
785      if (pageRequest != null)
786      {
787        // Indicate no more pages.
788        Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
789        searchOperation.getResponseControls().add(control);
790      }
791
792      return;
793    }
794
795    // Check whether the client requested debug information about the
796    // contribution of the indexes to the search.
797    StringBuilder debugBuffer = null;
798    if (searchOperation.getAttributes().contains(ATTR_DEBUG_SEARCH_INDEX))
799    {
800      debugBuffer = new StringBuilder();
801    }
802
803    EntryIDSet entryIDList = null;
804    boolean candidatesAreInScope = false;
805    if(sortRequest != null)
806    {
807      for(VLVIndex vlvIndex : vlvIndexMap.values())
808      {
809        try
810        {
811          entryIDList = vlvIndex.evaluate(null, searchOperation, sortRequest, vlvRequest, debugBuffer);
812          if(entryIDList != null)
813          {
814            searchOperation.addResponseControl(new ServerSideSortResponseControl(SUCCESS, null));
815            candidatesAreInScope = true;
816            break;
817          }
818        }
819        catch (DirectoryException de)
820        {
821          searchOperation.addResponseControl(new ServerSideSortResponseControl(de.getResultCode().intValue(), null));
822
823          if (sortRequest.isCritical())
824          {
825            throw de;
826          }
827        }
828      }
829    }
830
831    if(entryIDList == null)
832    {
833      // See if we could use a virtual attribute rule to process the search.
834      for (VirtualAttributeRule rule : DirectoryServer.getVirtualAttributes())
835      {
836        if (rule.getProvider().isSearchable(rule, searchOperation, true))
837        {
838          rule.getProvider().processSearch(rule, searchOperation);
839          return;
840        }
841      }
842
843      // Create an index filter to get the search result candidate entries.
844      IndexFilter indexFilter =
845        new IndexFilter(this, searchOperation, debugBuffer,
846            rootContainer.getMonitorProvider());
847
848      // Evaluate the filter against the attribute indexes.
849      entryIDList = indexFilter.evaluate();
850
851      // Evaluate the search scope against the id2children and id2subtree
852      // indexes.
853      if (entryIDList.size() > IndexFilter.FILTER_CANDIDATE_THRESHOLD)
854      {
855        // Read the ID from dn2id.
856        EntryID baseID = dn2id.get(null, aBaseDN, LockMode.DEFAULT);
857        if (baseID == null)
858        {
859          throw new DirectoryException(
860              ResultCode.NO_SUCH_OBJECT, ERR_SEARCH_NO_SUCH_OBJECT.get(aBaseDN), getMatchedDN(aBaseDN), null);
861        }
862        DatabaseEntry baseIDData = baseID.getDatabaseEntry();
863
864        EntryIDSet scopeList;
865        if (searchScope == SearchScope.SINGLE_LEVEL)
866        {
867          scopeList = id2children.readKey(baseIDData, null, LockMode.DEFAULT);
868        }
869        else
870        {
871          scopeList = id2subtree.readKey(baseIDData, null, LockMode.DEFAULT);
872          if (searchScope == SearchScope.WHOLE_SUBTREE)
873          {
874            // The id2subtree list does not include the base entry ID.
875            scopeList.add(baseID);
876          }
877        }
878        entryIDList.retainAll(scopeList);
879        if (debugBuffer != null)
880        {
881          debugBuffer.append(" scope=");
882          debugBuffer.append(searchScope);
883          scopeList.toString(debugBuffer);
884        }
885        if (scopeList.isDefined())
886        {
887          // In this case we know that every candidate is in scope.
888          candidatesAreInScope = true;
889        }
890      }
891
892      if (sortRequest != null)
893      {
894          try
895          {
896            //If the sort key is not present, the sorting will generate the
897            //default ordering. VLV search request goes through as if
898            //this sort key was not found in the user entry.
899            entryIDList = EntryIDSetSorter.sort(this, entryIDList,
900                searchOperation,
901                sortRequest.getSortOrder(),
902                vlvRequest);
903            if(sortRequest.containsSortKeys())
904            {
905              searchOperation.addResponseControl(new ServerSideSortResponseControl(SUCCESS, null));
906            }
907            else
908            {
909              /*
910               * There is no sort key associated with the sort control. Since it
911               * came here it means that the criticality is false so let the
912               * server return all search results unsorted and include the
913               * sortKeyResponseControl in the searchResultDone message.
914               */
915              searchOperation.addResponseControl(new ServerSideSortResponseControl(NO_SUCH_ATTRIBUTE, null));
916            }
917          }
918          catch (DirectoryException de)
919          {
920            searchOperation.addResponseControl(new ServerSideSortResponseControl(de.getResultCode().intValue(), null));
921
922            if (sortRequest.isCritical())
923            {
924              throw de;
925            }
926          }
927        }
928      }
929
930    // If requested, construct and return a fictitious entry containing
931    // debug information, and no other entries.
932    if (debugBuffer != null)
933    {
934      debugBuffer.append(" final=");
935      entryIDList.toString(debugBuffer);
936
937      Attribute attr = Attributes.create(ATTR_DEBUG_SEARCH_INDEX, debugBuffer.toString());
938      Entry debugEntry = new Entry(DN.valueOf("cn=debugsearch"), null, null, null);
939      debugEntry.addAttribute(attr, new ArrayList<ByteString>());
940
941      searchOperation.returnEntry(debugEntry, null);
942      return;
943    }
944
945    if (entryIDList.isDefined())
946    {
947      if(rootContainer.getMonitorProvider().isFilterUseEnabled())
948      {
949        rootContainer.getMonitorProvider().updateIndexedSearchCount();
950      }
951      searchIndexed(entryIDList, candidatesAreInScope, searchOperation, pageRequest);
952    }
953    else
954    {
955      if(rootContainer.getMonitorProvider().isFilterUseEnabled())
956      {
957        rootContainer.getMonitorProvider().updateUnindexedSearchCount();
958      }
959
960      searchOperation.addAdditionalLogItem(keyOnly(getClass(), "unindexed"));
961
962      // See if we could use a virtual attribute rule to process the search.
963      for (VirtualAttributeRule rule : DirectoryServer.getVirtualAttributes())
964      {
965        if (rule.getProvider().isSearchable(rule, searchOperation, false))
966        {
967          rule.getProvider().processSearch(rule, searchOperation);
968          return;
969        }
970      }
971
972      ClientConnection clientConnection = searchOperation.getClientConnection();
973      if (!clientConnection.hasPrivilege(Privilege.UNINDEXED_SEARCH, searchOperation))
974      {
975        throw new DirectoryException(
976            ResultCode.INSUFFICIENT_ACCESS_RIGHTS, ERR_SEARCH_UNINDEXED_INSUFFICIENT_PRIVILEGES.get());
977      }
978
979      if (sortRequest != null)
980      {
981        // FIXME -- Add support for sorting unindexed searches using indexes
982        //          like DSEE currently does.
983        searchOperation.addResponseControl(new ServerSideSortResponseControl(UNWILLING_TO_PERFORM, null));
984
985        if (sortRequest.isCritical())
986        {
987          throw new DirectoryException(
988              ResultCode.UNAVAILABLE_CRITICAL_EXTENSION, ERR_SEARCH_CANNOT_SORT_UNINDEXED.get());
989        }
990      }
991
992      searchNotIndexed(searchOperation, pageRequest);
993    }
994  }
995
996  /**
997   * We were not able to obtain a set of candidate entry IDs for the
998   * search from the indexes.
999   * <p>
1000   * Here we are relying on the DN key order to ensure children are
1001   * returned after their parents.
1002   * <ul>
1003   * <li>iterate through a subtree range of the DN database
1004   * <li>discard non-children DNs if the search scope is single level
1005   * <li>fetch the entry by ID from the entry cache or the entry database
1006   * <li>return the entry if it matches the filter
1007   * </ul>
1008   *
1009   * @param searchOperation The search operation.
1010   * @param pageRequest A Paged Results control, or null if none.
1011   * @throws DirectoryException If an error prevented the search from being
1012   * processed.
1013   */
1014  private void searchNotIndexed(SearchOperation searchOperation, PagedResultsControl pageRequest)
1015      throws DirectoryException, CanceledOperationException
1016  {
1017    DN aBaseDN = searchOperation.getBaseDN();
1018    SearchScope searchScope = searchOperation.getScope();
1019    boolean manageDsaIT = isManageDsaITOperation(searchOperation);
1020
1021    // The base entry must already have been processed if this is
1022    // a request for the next page in paged results.  So we skip
1023    // the base entry processing if the cookie is set.
1024    if (pageRequest == null || pageRequest.getCookie().length() == 0)
1025    {
1026      // Fetch the base entry.
1027      Entry baseEntry = fetchBaseEntry(aBaseDN, searchScope);
1028
1029      if (!manageDsaIT)
1030      {
1031        dn2uri.checkTargetForReferral(baseEntry, searchScope);
1032      }
1033
1034      /*
1035       * The base entry is only included for whole subtree search.
1036       */
1037      if (searchScope == SearchScope.WHOLE_SUBTREE
1038          && searchOperation.getFilter().matchesEntry(baseEntry))
1039      {
1040        searchOperation.returnEntry(baseEntry, null);
1041      }
1042
1043      if (!manageDsaIT
1044          && !dn2uri.returnSearchReferences(searchOperation)
1045          && pageRequest != null)
1046      {
1047        // Indicate no more pages.
1048        Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
1049        searchOperation.getResponseControls().add(control);
1050      }
1051    }
1052
1053    /*
1054     * We will iterate forwards through a range of the dn2id keys to
1055     * find subordinates of the target entry from the top of the tree
1056     * downwards. For example, any subordinates of "dc=example,dc=com" appear
1057     * in dn2id with a key ending in ",dc=example,dc=com". The entry
1058     * "cn=joe,ou=people,dc=example,dc=com" will appear after the entry
1059     * "ou=people,dc=example,dc=com".
1060     */
1061    byte[] baseDNKey = dnToDNKey(aBaseDN, this.baseDN.size());
1062    final byte special = 0x00;
1063    byte[] suffix = Arrays.copyOf(baseDNKey, baseDNKey.length+1);
1064    suffix[suffix.length - 1] = special;
1065
1066    /*
1067     * Set the ending value to a value of equal length but slightly
1068     * greater than the suffix. Since keys are compared in
1069     * reverse order we must set the first byte (the comma).
1070     * No possibility of overflow here.
1071     */
1072    byte[] end = Arrays.copyOf(suffix, suffix.length);
1073    end[end.length - 1] = special + 1;
1074
1075    // Set the starting value.
1076    byte[] begin;
1077    if (pageRequest != null && pageRequest.getCookie().length() != 0)
1078    {
1079      // The cookie contains the DN of the next entry to be returned.
1080      try
1081      {
1082        begin = pageRequest.getCookie().toByteArray();
1083      }
1084      catch (Exception e)
1085      {
1086        logger.traceException(e);
1087        LocalizableMessage msg = ERR_INVALID_PAGED_RESULTS_COOKIE.get(pageRequest.getCookie().toHexString());
1088        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, msg, e);
1089      }
1090    }
1091    else
1092    {
1093      // Set the starting value to the suffix.
1094      begin = suffix;
1095    }
1096
1097    DatabaseEntry data = new DatabaseEntry();
1098    DatabaseEntry key = new DatabaseEntry(begin);
1099
1100    int lookthroughCount = 0;
1101    int lookthroughLimit = searchOperation.getClientConnection().getLookthroughLimit();
1102
1103    try
1104    {
1105      Cursor cursor = dn2id.openCursor(null, null);
1106      try
1107      {
1108        // Initialize the cursor very close to the starting value.
1109        OperationStatus status = cursor.getSearchKeyRange(key, data, LockMode.DEFAULT);
1110
1111        // Step forward until we pass the ending value.
1112        while (status == OperationStatus.SUCCESS)
1113        {
1114          if(lookthroughLimit > 0 && lookthroughCount > lookthroughLimit)
1115          {
1116            //Lookthrough limit exceeded
1117            searchOperation.setResultCode(ResultCode.ADMIN_LIMIT_EXCEEDED);
1118            searchOperation.appendErrorMessage(NOTE_LOOKTHROUGH_LIMIT_EXCEEDED.get(lookthroughLimit));
1119            return;
1120          }
1121          int cmp = dn2id.getComparator().compare(key.getData(), end);
1122          if (cmp >= 0)
1123          {
1124            // We have gone past the ending value.
1125            break;
1126          }
1127
1128          // We have found a subordinate entry.
1129
1130          EntryID entryID = new EntryID(data);
1131
1132          boolean isInScope =
1133              searchScope != SearchScope.SINGLE_LEVEL
1134                  // Check if this entry is an immediate child.
1135                  || findDNKeyParent(key.getData()) == baseDNKey.length;
1136          if (isInScope)
1137          {
1138            // Process the candidate entry.
1139            final Entry entry = getEntry(entryID);
1140            if (entry != null)
1141            {
1142              lookthroughCount++;
1143
1144              if ((manageDsaIT || entry.getReferralURLs() == null)
1145                  && searchOperation.getFilter().matchesEntry(entry))
1146              {
1147                if (pageRequest != null
1148                    && searchOperation.getEntriesSent() == pageRequest.getSize())
1149                {
1150                  // The current page is full.
1151                  // Set the cookie to remember where we were.
1152                  ByteString cookie = ByteString.wrap(key.getData());
1153                  Control control = new PagedResultsControl(pageRequest.isCritical(), 0, cookie);
1154                  searchOperation.getResponseControls().add(control);
1155                  return;
1156                }
1157
1158                if (!searchOperation.returnEntry(entry, null))
1159                {
1160                  // We have been told to discontinue processing of the
1161                  // search. This could be due to size limit exceeded or
1162                  // operation cancelled.
1163                  return;
1164                }
1165              }
1166            }
1167          }
1168
1169          searchOperation.checkIfCanceled(false);
1170
1171          // Move to the next record.
1172          status = cursor.getNext(key, data, LockMode.DEFAULT);
1173        }
1174      }
1175      finally
1176      {
1177        cursor.close();
1178      }
1179    }
1180    catch (DatabaseException e)
1181    {
1182      logger.traceException(e);
1183    }
1184
1185    if (pageRequest != null)
1186    {
1187      // Indicate no more pages.
1188      Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
1189      searchOperation.getResponseControls().add(control);
1190    }
1191  }
1192
1193  /**
1194   * Returns the entry corresponding to the provided entryID.
1195   *
1196   * @param entryID
1197   *          the id of the entry to retrieve
1198   * @return the entry corresponding to the provided entryID
1199   * @throws DirectoryException
1200   *           If an error occurs retrieving the entry
1201   */
1202  Entry getEntry(EntryID entryID) throws DirectoryException
1203  {
1204    // Try the entry cache first.
1205    final EntryCache<?> entryCache = getEntryCache();
1206    final Entry cacheEntry = entryCache.getEntry(backendID, entryID.longValue());
1207    if (cacheEntry != null)
1208    {
1209      return cacheEntry;
1210    }
1211
1212    final Entry entry = id2entry.get(null, entryID, LockMode.DEFAULT);
1213    if (entry != null)
1214    {
1215      // Put the entry in the cache making sure not to overwrite a newer copy
1216      // that may have been inserted since the time we read the cache.
1217      entryCache.putEntryIfAbsent(entry, backendID, entryID.longValue());
1218    }
1219    return entry;
1220  }
1221
1222  /**
1223   * We were able to obtain a set of candidate entry IDs for the
1224   * search from the indexes.
1225   * <p>
1226   * Here we are relying on ID order to ensure children are returned
1227   * after their parents.
1228   * <ul>
1229   * <li>Iterate through the candidate IDs
1230   * <li>fetch entry by ID from cache or id2entry
1231   * <li>put the entry in the cache if not present
1232   * <li>discard entries that are not in scope
1233   * <li>return entry if it matches the filter
1234   * </ul>
1235   *
1236   * @param entryIDList The candidate entry IDs.
1237   * @param candidatesAreInScope true if it is certain that every candidate
1238   *                             entry is in the search scope.
1239   * @param searchOperation The search operation.
1240   * @param pageRequest A Paged Results control, or null if none.
1241   * @throws DirectoryException If an error prevented the search from being
1242   * processed.
1243   */
1244  private void searchIndexed(EntryIDSet entryIDList,
1245      boolean candidatesAreInScope,
1246      SearchOperation searchOperation,
1247      PagedResultsControl pageRequest)
1248  throws DirectoryException, CanceledOperationException
1249  {
1250    SearchScope searchScope = searchOperation.getScope();
1251    DN aBaseDN = searchOperation.getBaseDN();
1252    boolean manageDsaIT = isManageDsaITOperation(searchOperation);
1253    boolean continueSearch = true;
1254
1255    // Set the starting value.
1256    EntryID begin = null;
1257    if (pageRequest != null && pageRequest.getCookie().length() != 0)
1258    {
1259      // The cookie contains the ID of the next entry to be returned.
1260      try
1261      {
1262        begin = new EntryID(pageRequest.getCookie().toLong());
1263      }
1264      catch (Exception e)
1265      {
1266        logger.traceException(e);
1267        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM,
1268            ERR_INVALID_PAGED_RESULTS_COOKIE.get(pageRequest.getCookie().toHexString()), e);
1269      }
1270    }
1271    else if (!manageDsaIT)
1272    {
1273      // Return any search result references.
1274      continueSearch = dn2uri.returnSearchReferences(searchOperation);
1275    }
1276
1277    // Make sure the candidate list is smaller than the lookthrough limit
1278    int lookthroughLimit =
1279      searchOperation.getClientConnection().getLookthroughLimit();
1280    if(lookthroughLimit > 0 && entryIDList.size() > lookthroughLimit)
1281    {
1282      //Lookthrough limit exceeded
1283      searchOperation.setResultCode(ResultCode.ADMIN_LIMIT_EXCEEDED);
1284      searchOperation.appendErrorMessage(NOTE_LOOKTHROUGH_LIMIT_EXCEEDED.get(lookthroughLimit));
1285      continueSearch = false;
1286    }
1287
1288    // Iterate through the index candidates.
1289    if (continueSearch)
1290    {
1291      for (Iterator<EntryID> it = entryIDList.iterator(begin); it.hasNext();)
1292      {
1293        final EntryID id = it.next();
1294
1295        Entry entry;
1296        try
1297        {
1298          entry = getEntry(id);
1299        }
1300        catch (Exception e)
1301        {
1302          logger.traceException(e);
1303          continue;
1304        }
1305
1306        // Process the candidate entry.
1307        if (entry != null)
1308        {
1309          // Filter the entry if it is in scope.
1310          if (isInScope(candidatesAreInScope, searchScope, aBaseDN, entry)
1311              && (manageDsaIT || entry.getReferralURLs() == null)
1312              && searchOperation.getFilter().matchesEntry(entry))
1313          {
1314            if (pageRequest != null
1315                && searchOperation.getEntriesSent() == pageRequest.getSize())
1316            {
1317              // The current page is full.
1318              // Set the cookie to remember where we were.
1319              byte[] cookieBytes = id.getDatabaseEntry().getData();
1320              ByteString cookie = ByteString.wrap(cookieBytes);
1321              Control control = new PagedResultsControl(pageRequest.isCritical(), 0, cookie);
1322              searchOperation.getResponseControls().add(control);
1323              return;
1324            }
1325
1326            if (!searchOperation.returnEntry(entry, null))
1327            {
1328              // We have been told to discontinue processing of the
1329              // search. This could be due to size limit exceeded or
1330              // operation cancelled.
1331              break;
1332            }
1333          }
1334        }
1335      }
1336      searchOperation.checkIfCanceled(false);
1337    }
1338
1339    // Before we return success from the search we must ensure the base entry
1340    // exists. However, if we have returned at least one entry or subordinate
1341    // reference it implies the base does exist, so we can omit the check.
1342    if (searchOperation.getEntriesSent() == 0
1343        && searchOperation.getReferencesSent() == 0)
1344    {
1345      // Fetch the base entry if it exists.
1346      Entry baseEntry = fetchBaseEntry(aBaseDN, searchScope);
1347
1348      if (!manageDsaIT)
1349      {
1350        dn2uri.checkTargetForReferral(baseEntry, searchScope);
1351      }
1352    }
1353
1354    if (pageRequest != null)
1355    {
1356      // Indicate no more pages.
1357      Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
1358      searchOperation.getResponseControls().add(control);
1359    }
1360  }
1361
1362  private boolean isInScope(boolean candidatesAreInScope, SearchScope searchScope, DN aBaseDN, Entry entry)
1363  {
1364    DN entryDN = entry.getName();
1365
1366    if (candidatesAreInScope)
1367    {
1368      return true;
1369    }
1370    else if (searchScope == SearchScope.SINGLE_LEVEL)
1371    {
1372      // Check if this entry is an immediate child.
1373      if (entryDN.size() == aBaseDN.size() + 1
1374          && entryDN.isDescendantOf(aBaseDN))
1375      {
1376        return true;
1377      }
1378    }
1379    else if (searchScope == SearchScope.WHOLE_SUBTREE)
1380    {
1381      if (entryDN.isDescendantOf(aBaseDN))
1382      {
1383        return true;
1384      }
1385    }
1386    else if (searchScope == SearchScope.SUBORDINATES
1387        && entryDN.size() > aBaseDN.size()
1388        && entryDN.isDescendantOf(aBaseDN))
1389    {
1390      return true;
1391    }
1392    return false;
1393  }
1394
1395  /**
1396   * Adds the provided entry to this database.  This method must ensure that the
1397   * entry is appropriate for the database and that no entry already exists with
1398   * the same DN.  The caller must hold a write lock on the DN of the provided
1399   * entry.
1400   *
1401   * @param entry        The entry to add to this database.
1402   * @param addOperation The add operation with which the new entry is
1403   *                     associated.  This may be <CODE>null</CODE> for adds
1404   *                     performed internally.
1405   * @throws DirectoryException If a problem occurs while trying to add the
1406   *                            entry.
1407   * @throws DatabaseException If an error occurs in the JE database.
1408   * @throws CanceledOperationException if this operation should be cancelled.
1409   */
1410  void addEntry(Entry entry, AddOperation addOperation)
1411  throws DatabaseException, DirectoryException, CanceledOperationException
1412  {
1413    Transaction txn = beginTransaction();
1414    DN parentDN = getParentWithinBase(entry.getName());
1415
1416    try
1417    {
1418      // Check whether the entry already exists.
1419      if (dn2id.get(txn, entry.getName(), LockMode.DEFAULT) != null)
1420      {
1421        throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS,
1422            ERR_ADD_ENTRY_ALREADY_EXISTS.get(entry.getName()));
1423      }
1424
1425      // Check that the parent entry exists.
1426      EntryID parentID = null;
1427      if (parentDN != null)
1428      {
1429        // Check for referral entries above the target.
1430        dn2uri.targetEntryReferrals(entry.getName(), null);
1431
1432        // Read the parent ID from dn2id.
1433        parentID = dn2id.get(txn, parentDN, LockMode.DEFAULT);
1434        if (parentID == null)
1435        {
1436          throw new DirectoryException(ResultCode.NO_SUCH_OBJECT,
1437              ERR_ADD_NO_SUCH_OBJECT.get(entry.getName()), getMatchedDN(baseDN), null);
1438        }
1439      }
1440
1441      EntryID entryID = rootContainer.getNextEntryID();
1442
1443      // Insert into dn2id.
1444      if (!dn2id.insert(txn, entry.getName(), entryID))
1445      {
1446        // Do not ever expect to come through here.
1447        throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS,
1448            ERR_ADD_ENTRY_ALREADY_EXISTS.get(entry.getName()));
1449      }
1450
1451      // Update the referral database for referral entries.
1452      if (!dn2uri.addEntry(txn, entry))
1453      {
1454        // Do not ever expect to come through here.
1455        throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS,
1456            ERR_ADD_ENTRY_ALREADY_EXISTS.get(entry.getName()));
1457      }
1458
1459      // Insert into id2entry.
1460      if (!id2entry.insert(txn, entryID, entry))
1461      {
1462        // Do not ever expect to come through here.
1463        throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS,
1464            ERR_ADD_ENTRY_ALREADY_EXISTS.get(entry.getName()));
1465      }
1466
1467      // Insert into the indexes, in index configuration order.
1468      final IndexBuffer indexBuffer = new IndexBuffer(this);
1469      indexInsertEntry(indexBuffer, entry, entryID);
1470
1471      // Insert into id2children and id2subtree.
1472      // The database transaction locks on these records will be hotly
1473      // contested so we do them last so as to hold the locks for the
1474      // shortest duration.
1475      if (parentDN != null)
1476      {
1477        final ByteString parentIDKeyBytes = toByteString(parentID);
1478        id2children.insertID(indexBuffer, parentIDKeyBytes, entryID);
1479        id2subtree.insertID(indexBuffer, parentIDKeyBytes, entryID);
1480
1481        // Iterate up through the superior entries, starting above the parent.
1482        for (DN dn = getParentWithinBase(parentDN); dn != null;
1483        dn = getParentWithinBase(dn))
1484        {
1485          // Read the ID from dn2id.
1486          EntryID nodeID = dn2id.get(txn, dn, LockMode.DEFAULT);
1487          if (nodeID == null)
1488          {
1489            throw new JebException(ERR_MISSING_DN2ID_RECORD.get(dn));
1490          }
1491
1492          // Insert into id2subtree for this node.
1493          id2subtree.insertID(indexBuffer, toByteString(nodeID), entryID);
1494        }
1495      }
1496      indexBuffer.flush(txn);
1497
1498      if(addOperation != null)
1499      {
1500        // One last check before committing
1501        addOperation.checkIfCanceled(true);
1502      }
1503
1504      // Commit the transaction.
1505      EntryContainer.transactionCommit(txn);
1506
1507      // Update the entry cache.
1508      EntryCache<?> entryCache = DirectoryServer.getEntryCache();
1509      if (entryCache != null)
1510      {
1511        entryCache.putEntry(entry, backendID, entryID.longValue());
1512      }
1513    }
1514    catch (DatabaseException | DirectoryException | CanceledOperationException e)
1515    {
1516      EntryContainer.transactionAbort(txn);
1517      throw e;
1518    }
1519    catch (Exception e)
1520    {
1521      EntryContainer.transactionAbort(txn);
1522
1523      String msg = e.getMessage();
1524      if (msg == null)
1525      {
1526        msg = stackTraceToSingleLineString(e);
1527      }
1528      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1529          ERR_UNCHECKED_EXCEPTION.get(msg), e);
1530    }
1531  }
1532
1533  private ByteString toByteString(EntryID entryID)
1534  {
1535    return ByteString.wrap(entryID.getDatabaseEntry().getData());
1536  }
1537
1538  /**
1539   * Removes the specified entry from this database.  This method must ensure
1540   * that the entry exists and that it does not have any subordinate entries
1541   * (unless the database supports a subtree delete operation and the client
1542   * included the appropriate information in the request).  The caller must hold
1543   * a write lock on the provided entry DN.
1544   *
1545   * @param entryDN         The DN of the entry to remove from this database.
1546   * @param deleteOperation The delete operation with which this action is
1547   *                        associated.  This may be <CODE>null</CODE> for
1548   *                        deletes performed internally.
1549   * @throws DirectoryException If a problem occurs while trying to remove the
1550   *                            entry.
1551   * @throws DatabaseException If an error occurs in the JE database.
1552   * @throws CanceledOperationException if this operation should be cancelled.
1553   */
1554  void deleteEntry(DN entryDN, DeleteOperation deleteOperation)
1555  throws DirectoryException, DatabaseException, CanceledOperationException
1556  {
1557    Transaction txn = beginTransaction();
1558    final IndexBuffer indexBuffer = new IndexBuffer(this);
1559
1560    try
1561    {
1562      // Check for referral entries above the target entry.
1563      dn2uri.targetEntryReferrals(entryDN, null);
1564
1565      // Determine whether this is a subtree delete.
1566      boolean isSubtreeDelete = deleteOperation != null
1567          && deleteOperation.getRequestControl(SubtreeDeleteControl.DECODER) != null;
1568
1569      /*
1570       * We will iterate forwards through a range of the dn2id keys to
1571       * find subordinates of the target entry from the top of the tree
1572       * downwards.
1573       */
1574      byte[] entryDNKey = dnToDNKey(entryDN, this.baseDN.size());
1575      byte special = 0x00;
1576      byte[] suffix = Arrays.copyOf(entryDNKey, entryDNKey.length+1);
1577      suffix[suffix.length - 1] = special;
1578
1579      /*
1580       * Set the ending value to a value of equal length but slightly
1581       * greater than the suffix.
1582       */
1583      byte[] end = Arrays.copyOf(suffix, suffix.length);
1584      end[end.length - 1] = (byte) (special + 1);
1585
1586      int subordinateEntriesDeleted = 0;
1587
1588      DatabaseEntry data = new DatabaseEntry();
1589      DatabaseEntry key = new DatabaseEntry(suffix);
1590
1591      CursorConfig cursorConfig = new CursorConfig();
1592      cursorConfig.setReadCommitted(true);
1593      Cursor cursor = dn2id.openCursor(txn, cursorConfig);
1594      try
1595      {
1596        // Initialize the cursor very close to the starting value.
1597        OperationStatus status = cursor.getSearchKeyRange(key, data, LockMode.DEFAULT);
1598
1599        // Step forward until the key is greater than the starting value.
1600        while (status == OperationStatus.SUCCESS &&
1601            dn2id.getComparator().compare(key.getData(), suffix) <= 0)
1602        {
1603          status = cursor.getNext(key, data, LockMode.DEFAULT);
1604        }
1605
1606        // Step forward until we pass the ending value.
1607        while (status == OperationStatus.SUCCESS)
1608        {
1609          int cmp = dn2id.getComparator().compare(key.getData(), end);
1610          if (cmp >= 0)
1611          {
1612            // We have gone past the ending value.
1613            break;
1614          }
1615
1616          // We have found a subordinate entry.
1617          if (!isSubtreeDelete)
1618          {
1619            // The subtree delete control was not specified and
1620            // the target entry is not a leaf.
1621            throw new DirectoryException(ResultCode.NOT_ALLOWED_ON_NONLEAF,
1622                ERR_DELETE_NOT_ALLOWED_ON_NONLEAF.get(entryDN));
1623          }
1624
1625          /*
1626           * Delete this entry which by now must be a leaf because
1627           * we have been deleting from the bottom of the tree upwards.
1628           */
1629          EntryID entryID = new EntryID(data);
1630
1631          // Invoke any subordinate delete plugins on the entry.
1632          if (deleteOperation != null
1633              && !deleteOperation.isSynchronizationOperation())
1634          {
1635            Entry subordinateEntry = id2entry.get(txn, entryID, LockMode.DEFAULT);
1636            SubordinateDelete pluginResult =
1637              getPluginConfigManager().invokeSubordinateDeletePlugins(
1638                  deleteOperation, subordinateEntry);
1639
1640            if (!pluginResult.continueProcessing())
1641            {
1642              throw new DirectoryException(
1643                  DirectoryServer.getServerErrorResultCode(),
1644                  ERR_DELETE_ABORTED_BY_SUBORDINATE_PLUGIN.get(subordinateEntry.getName()));
1645            }
1646          }
1647
1648          deleteEntry(txn, indexBuffer, true, entryDN, key, entryID);
1649          subordinateEntriesDeleted++;
1650
1651          if(deleteOperation != null)
1652          {
1653            deleteOperation.checkIfCanceled(false);
1654          }
1655
1656          // Get the next DN.
1657          data = new DatabaseEntry();
1658          status = cursor.getNext(key, data, LockMode.DEFAULT);
1659        }
1660      }
1661      finally
1662      {
1663        cursor.close();
1664      }
1665
1666      // draft-armijo-ldap-treedelete, 4.1 Tree Delete Semantics:
1667      // The server MUST NOT chase referrals stored in the tree.  If
1668      // information about referrals is stored in this section of the
1669      // tree, this pointer will be deleted.
1670      deleteEntry(txn, indexBuffer,
1671          isSubtreeDelete || isManageDsaITOperation(deleteOperation),
1672          entryDN, null, null);
1673
1674      indexBuffer.flush(txn);
1675
1676
1677      if(deleteOperation != null)
1678      {
1679        // One last check before committing
1680        deleteOperation.checkIfCanceled(true);
1681      }
1682
1683      // Commit the transaction.
1684      EntryContainer.transactionCommit(txn);
1685
1686      if(isSubtreeDelete)
1687      {
1688        deleteOperation.addAdditionalLogItem(
1689            unquotedKeyValue(getClass(), "deletedEntries",
1690                subordinateEntriesDeleted + 1));
1691      }
1692    }
1693    catch (DatabaseException | DirectoryException | CanceledOperationException e)
1694    {
1695      EntryContainer.transactionAbort(txn);
1696      throw e;
1697    }
1698    catch (Exception e)
1699    {
1700      EntryContainer.transactionAbort(txn);
1701
1702      String msg = e.getMessage();
1703      if (msg == null)
1704      {
1705        msg = stackTraceToSingleLineString(e);
1706      }
1707      LocalizableMessage message = ERR_UNCHECKED_EXCEPTION.get(msg);
1708      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1709          message, e);
1710    }
1711  }
1712
1713  private void deleteEntry(Transaction txn,
1714      IndexBuffer indexBuffer,
1715      boolean manageDsaIT,
1716      DN targetDN,
1717      DatabaseEntry leafDNKey,
1718      EntryID leafID)
1719  throws DatabaseException, DirectoryException, JebException
1720  {
1721    if(leafID == null || leafDNKey == null)
1722    {
1723      // Read the entry ID from dn2id.
1724      if(leafDNKey == null)
1725      {
1726        leafDNKey = new DatabaseEntry(dnToDNKey(targetDN, baseDN.size()));
1727      }
1728      DatabaseEntry value = new DatabaseEntry();
1729      OperationStatus status = dn2id.read(txn, leafDNKey, value, LockMode.RMW);
1730      if (status != OperationStatus.SUCCESS)
1731      {
1732        throw new DirectoryException(
1733            ResultCode.NO_SUCH_OBJECT, ERR_DELETE_NO_SUCH_OBJECT.get(leafDNKey), getMatchedDN(baseDN), null);
1734      }
1735      leafID = new EntryID(value);
1736    }
1737
1738    // Remove from dn2id.
1739    if (dn2id.delete(txn, leafDNKey) != OperationStatus.SUCCESS)
1740    {
1741      // Do not expect to ever come through here.
1742      LocalizableMessage message = ERR_DELETE_NO_SUCH_OBJECT.get(leafDNKey);
1743      DN matchedDN = getMatchedDN(baseDN);
1744      throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, message, matchedDN, null);
1745    }
1746
1747    // Check that the entry exists in id2entry and read its contents.
1748    Entry entry = id2entry.get(txn, leafID, LockMode.RMW);
1749    if (entry == null)
1750    {
1751      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1752          ERR_MISSING_ID2ENTRY_RECORD.get(leafID));
1753    }
1754
1755    if (!manageDsaIT)
1756    {
1757      dn2uri.checkTargetForReferral(entry, null);
1758    }
1759
1760    // Update the referral database.
1761    dn2uri.deleteEntry(txn, entry);
1762
1763    // Remove from id2entry.
1764    if (!id2entry.remove(txn, leafID))
1765    {
1766      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1767          ERR_MISSING_ID2ENTRY_RECORD.get(leafID));
1768    }
1769
1770    // Remove from the indexes, in index config order.
1771    indexRemoveEntry(indexBuffer, entry, leafID);
1772
1773    // Remove the id2c and id2s records for this entry.
1774    final ByteString leafIDKeyBytes = ByteString.valueOf(leafID.longValue());
1775    id2children.delete(indexBuffer, leafIDKeyBytes);
1776    id2subtree.delete(indexBuffer, leafIDKeyBytes);
1777
1778    // Iterate up through the superior entries from the target entry.
1779    boolean isParent = true;
1780    for (DN parentDN = getParentWithinBase(targetDN); parentDN != null;
1781    parentDN = getParentWithinBase(parentDN))
1782    {
1783      // Read the ID from dn2id.
1784      EntryID parentID = dn2id.get(txn, parentDN, LockMode.DEFAULT);
1785      if (parentID == null)
1786      {
1787        throw new JebException(ERR_MISSING_DN2ID_RECORD.get(parentDN));
1788      }
1789
1790      ByteString parentIDBytes = ByteString.valueOf(parentID.longValue());
1791      // Remove from id2children.
1792      if (isParent)
1793      {
1794        id2children.removeID(indexBuffer, parentIDBytes, leafID);
1795        isParent = false;
1796      }
1797      id2subtree.removeID(indexBuffer, parentIDBytes, leafID);
1798    }
1799
1800    // Remove the entry from the entry cache.
1801    EntryCache<?> entryCache = DirectoryServer.getEntryCache();
1802    if (entryCache != null)
1803    {
1804      entryCache.removeEntry(entry.getName());
1805    }
1806  }
1807
1808  /**
1809   * Indicates whether an entry with the specified DN exists.
1810   *
1811   * @param  entryDN  The DN of the entry for which to determine existence.
1812   *
1813   * @return  <CODE>true</CODE> if the specified entry exists,
1814   *          or <CODE>false</CODE> if it does not.
1815   *
1816   * @throws  DirectoryException  If a problem occurs while trying to make the
1817   *                              determination.
1818   */
1819  boolean entryExists(DN entryDN) throws DirectoryException
1820  {
1821    // Try the entry cache first.
1822    EntryCache<?> entryCache = DirectoryServer.getEntryCache();
1823    if (entryCache != null && entryCache.containsEntry(entryDN))
1824    {
1825      return true;
1826    }
1827
1828    try
1829    {
1830      return dn2id.get(null, entryDN, LockMode.DEFAULT) != null;
1831    }
1832    catch (DatabaseException e)
1833    {
1834      logger.traceException(e);
1835      return false;
1836    }
1837  }
1838
1839  /**
1840   * Fetch an entry by DN, trying the entry cache first, then the database. Retrieves the requested
1841   * entry, trying the entry cache first, then the database.
1842   *
1843   * @param entryDN
1844   *          The distinguished name of the entry to retrieve.
1845   * @return The requested entry, or <CODE>null</CODE> if the entry does not exist.
1846   * @throws DirectoryException
1847   *           If a problem occurs while trying to retrieve the entry.
1848   * @throws DatabaseException
1849   *           An error occurred during a database operation.
1850   */
1851  Entry getEntry(DN entryDN) throws DatabaseException, DirectoryException
1852  {
1853    EntryCache<?> entryCache = DirectoryServer.getEntryCache();
1854    if (entryCache != null)
1855    {
1856      Entry entry = entryCache.getEntry(entryDN);
1857      if (entry != null)
1858      {
1859        return entry;
1860      }
1861    }
1862
1863    EntryID entryID = dn2id.get(null, entryDN, LockMode.DEFAULT);
1864    if (entryID == null)
1865    {
1866      // The entryDN does not exist. Check for referral entries above the target entry.
1867      dn2uri.targetEntryReferrals(entryDN, null);
1868      return null;
1869    }
1870
1871    Entry entry = id2entry.get(null, entryID, LockMode.DEFAULT);
1872    if (entry != null && entryCache != null)
1873    {
1874      /*
1875       * Put the entry in the cache making sure not to overwrite a newer copy that may have been
1876       * inserted since the time we read the cache.
1877       */
1878      entryCache.putEntryIfAbsent(entry, backendID, entryID.longValue());
1879    }
1880    return entry;
1881  }
1882
1883  /**
1884   * The simplest case of replacing an entry in which the entry DN has
1885   * not changed.
1886   *
1887   * @param oldEntry           The old contents of the entry
1888   * @param newEntry           The new contents of the entry
1889   * @param modifyOperation The modify operation with which this action is
1890   *                        associated.  This may be <CODE>null</CODE> for
1891   *                        modifications performed internally.
1892   * @throws DatabaseException If an error occurs in the JE database.
1893   * @throws DirectoryException If a Directory Server error occurs.
1894   * @throws CanceledOperationException if this operation should be cancelled.
1895   */
1896  void replaceEntry(Entry oldEntry, Entry newEntry,
1897      ModifyOperation modifyOperation) throws DatabaseException,
1898      DirectoryException, CanceledOperationException
1899  {
1900    Transaction txn = beginTransaction();
1901
1902    try
1903    {
1904      // Read dn2id.
1905      EntryID entryID = dn2id.get(txn, newEntry.getName(), LockMode.RMW);
1906      if (entryID == null)
1907      {
1908        // The entry does not exist.
1909        throw new DirectoryException(ResultCode.NO_SUCH_OBJECT,
1910            ERR_MODIFY_NO_SUCH_OBJECT.get(newEntry.getName()), getMatchedDN(baseDN), null);
1911      }
1912
1913      if (!isManageDsaITOperation(modifyOperation))
1914      {
1915        // Check if the entry is a referral entry.
1916        dn2uri.checkTargetForReferral(oldEntry, null);
1917      }
1918
1919      // Update the referral database.
1920      if (modifyOperation != null)
1921      {
1922        // In this case we know from the operation what the modifications were.
1923        List<Modification> mods = modifyOperation.getModifications();
1924        dn2uri.modifyEntry(txn, oldEntry, newEntry, mods);
1925      }
1926      else
1927      {
1928        dn2uri.replaceEntry(txn, oldEntry, newEntry);
1929      }
1930
1931      // Replace id2entry.
1932      id2entry.put(txn, entryID, newEntry);
1933
1934      // Update the indexes.
1935      final IndexBuffer indexBuffer = new IndexBuffer(this);
1936      if (modifyOperation != null)
1937      {
1938        // In this case we know from the operation what the modifications were.
1939        List<Modification> mods = modifyOperation.getModifications();
1940        indexModifications(indexBuffer, oldEntry, newEntry, entryID, mods);
1941      }
1942      else
1943      {
1944        // The most optimal would be to figure out what the modifications were.
1945        indexRemoveEntry(indexBuffer, oldEntry, entryID);
1946        indexInsertEntry(indexBuffer, newEntry, entryID);
1947      }
1948
1949      indexBuffer.flush(txn);
1950
1951      if(modifyOperation != null)
1952      {
1953        // One last check before committing
1954        modifyOperation.checkIfCanceled(true);
1955      }
1956
1957      // Commit the transaction.
1958      EntryContainer.transactionCommit(txn);
1959
1960      // Update the entry cache.
1961      EntryCache<?> entryCache = DirectoryServer.getEntryCache();
1962      if (entryCache != null)
1963      {
1964        entryCache.putEntry(newEntry, backendID, entryID.longValue());
1965      }
1966    }
1967    catch (DatabaseException | DirectoryException | CanceledOperationException e)
1968    {
1969      EntryContainer.transactionAbort(txn);
1970      throw e;
1971    }
1972    catch (Exception e)
1973    {
1974      EntryContainer.transactionAbort(txn);
1975
1976      String msg = e.getMessage();
1977      if (msg == null)
1978      {
1979        msg = stackTraceToSingleLineString(e);
1980      }
1981      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1982          ERR_UNCHECKED_EXCEPTION.get(msg), e);
1983    }
1984  }
1985
1986  /**
1987   * Moves and/or renames the provided entry in this backend, altering any
1988   * subordinate entries as necessary.  This must ensure that an entry already
1989   * exists with the provided current DN, and that no entry exists with the
1990   * target DN of the provided entry.  The caller must hold write locks on both
1991   * the current DN and the new DN for the entry.
1992   *
1993   * @param currentDN         The current DN of the entry to be replaced.
1994   * @param entry             The new content to use for the entry.
1995   * @param modifyDNOperation The modify DN operation with which this action
1996   *                          is associated.  This may be <CODE>null</CODE>
1997   *                          for modify DN operations performed internally.
1998   * @throws DirectoryException
1999   *          If a problem occurs while trying to perform the rename.
2000   * @throws CanceledOperationException
2001   *          If this backend noticed and reacted to a request to cancel
2002   *          or abandon the modify DN operation.
2003   * @throws DatabaseException If an error occurs in the JE database.
2004   */
2005  void renameEntry(DN currentDN, Entry entry, ModifyDNOperation modifyDNOperation)
2006  throws DatabaseException, DirectoryException, CanceledOperationException
2007  {
2008    Transaction txn = beginTransaction();
2009    DN oldSuperiorDN = getParentWithinBase(currentDN);
2010    DN newSuperiorDN = getParentWithinBase(entry.getName());
2011    boolean isApexEntryMoved;
2012
2013    if(oldSuperiorDN != null)
2014    {
2015      isApexEntryMoved = ! oldSuperiorDN.equals(newSuperiorDN);
2016    }
2017    else if(newSuperiorDN != null)
2018    {
2019      isApexEntryMoved = ! newSuperiorDN.equals(oldSuperiorDN);
2020    }
2021    else
2022    {
2023      isApexEntryMoved = false;
2024    }
2025
2026    IndexBuffer buffer = new IndexBuffer(EntryContainer.this);
2027
2028    try
2029    {
2030      // Check whether the renamed entry already exists.
2031      if (!currentDN.equals(entry.getName()) &&
2032          dn2id.get(txn, entry.getName(), LockMode.DEFAULT) != null)
2033      {
2034        LocalizableMessage message = ERR_MODIFYDN_ALREADY_EXISTS.get(entry.getName());
2035        throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS, message);
2036      }
2037
2038      EntryID oldApexID = dn2id.get(txn, currentDN, LockMode.DEFAULT);
2039      if (oldApexID == null)
2040      {
2041        // Check for referral entries above the target entry.
2042        dn2uri.targetEntryReferrals(currentDN, null);
2043
2044        throw new DirectoryException(ResultCode.NO_SUCH_OBJECT,
2045            ERR_MODIFYDN_NO_SUCH_OBJECT.get(currentDN), getMatchedDN(baseDN), null);
2046      }
2047
2048      Entry oldApexEntry = id2entry.get(txn, oldApexID, LockMode.DEFAULT);
2049      if (oldApexEntry == null)
2050      {
2051        throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
2052            ERR_MISSING_ID2ENTRY_RECORD.get(oldApexID));
2053      }
2054
2055      if (!isManageDsaITOperation(modifyDNOperation))
2056      {
2057        dn2uri.checkTargetForReferral(oldApexEntry, null);
2058      }
2059
2060      EntryID newApexID = oldApexID;
2061      if (newSuperiorDN != null && isApexEntryMoved)
2062      {
2063        /*
2064         * We want to preserve the invariant that the ID of an
2065         * entry is greater than its parent, since search
2066         * results are returned in ID order.
2067         */
2068        EntryID newSuperiorID = dn2id.get(txn, newSuperiorDN, LockMode.DEFAULT);
2069        if (newSuperiorID == null)
2070        {
2071          throw new DirectoryException(ResultCode.NO_SUCH_OBJECT,
2072              ERR_NEW_SUPERIOR_NO_SUCH_OBJECT.get(newSuperiorDN), getMatchedDN(baseDN), null);
2073        }
2074
2075        if (newSuperiorID.compareTo(oldApexID) > 0)
2076        {
2077          // This move would break the above invariant so we must
2078          // renumber every entry that moves. This is even more
2079          // expensive since every entry has to be deleted from
2080          // and added back into the attribute indexes.
2081          newApexID = rootContainer.getNextEntryID();
2082
2083          if(logger.isTraceEnabled())
2084          {
2085            logger.trace("Move of target entry requires renumbering" +
2086                "all entries in the subtree. " +
2087                "Old DN: %s " +
2088                "New DN: %s " +
2089                "Old entry ID: %d " +
2090                "New entry ID: %d " +
2091                "New Superior ID: %d" +
2092                oldApexEntry.getName(), entry.getName(),
2093                oldApexID.longValue(), newApexID.longValue(),
2094                newSuperiorID.longValue());
2095          }
2096        }
2097      }
2098
2099      MovedEntry head = new MovedEntry(null, null, false);
2100      MovedEntry current = head;
2101      // Move or rename the apex entry.
2102      removeApexEntry(txn, buffer, oldSuperiorDN, oldApexID,
2103          newApexID, oldApexEntry, entry,isApexEntryMoved, modifyDNOperation,
2104          current);
2105      current = current.next;
2106
2107      /*
2108       * We will iterate forwards through a range of the dn2id keys to
2109       * find subordinates of the target entry from the top of the tree
2110       * downwards.
2111       */
2112      byte[] currentDNKey = dnToDNKey(currentDN, this.baseDN.size());
2113      byte special = 0x00;
2114      byte[] suffix = Arrays.copyOf(currentDNKey, currentDNKey.length+1);
2115      suffix[suffix.length - 1] = special;
2116
2117      /*
2118       * Set the ending value to a value of equal length but slightly
2119       * greater than the suffix.
2120       */
2121      byte[] end = Arrays.copyOf(suffix, suffix.length);
2122      end[end.length - 1] = (byte) (special + 1);
2123
2124      DatabaseEntry data = new DatabaseEntry();
2125      DatabaseEntry key = new DatabaseEntry(suffix);
2126
2127      CursorConfig cursorConfig = new CursorConfig();
2128      cursorConfig.setReadCommitted(true);
2129      Cursor cursor = dn2id.openCursor(txn, cursorConfig);
2130      try
2131      {
2132        // Initialize the cursor very close to the starting value.
2133        OperationStatus status = cursor.getSearchKeyRange(key, data, LockMode.DEFAULT);
2134
2135        // Step forward until the key is greater than the starting value.
2136        while (status == OperationStatus.SUCCESS &&
2137            dn2id.getComparator().compare(key.getData(), suffix) <= 0)
2138        {
2139          status = cursor.getNext(key, data, LockMode.DEFAULT);
2140        }
2141
2142        // Step forward until we pass the ending value.
2143        while (status == OperationStatus.SUCCESS)
2144        {
2145          int cmp = dn2id.getComparator().compare(key.getData(), end);
2146          if (cmp >= 0)
2147          {
2148            // We have gone past the ending value.
2149            break;
2150          }
2151
2152          // We have found a subordinate entry.
2153          EntryID oldID = new EntryID(data);
2154          Entry oldEntry = id2entry.get(txn, oldID, LockMode.DEFAULT);
2155
2156          // Construct the new DN of the entry.
2157          DN newDN = modDN(oldEntry.getName(),
2158              currentDN.size(),
2159              entry.getName());
2160
2161          // Assign a new entry ID if we are renumbering.
2162          EntryID newID = oldID;
2163          if (!newApexID.equals(oldApexID))
2164          {
2165            newID = rootContainer.getNextEntryID();
2166
2167            if(logger.isTraceEnabled())
2168            {
2169              logger.trace("Move of subordinate entry requires " +
2170                  "renumbering. " +
2171                  "Old DN: %s " +
2172                  "New DN: %s " +
2173                  "Old entry ID: %d " +
2174                  "New entry ID: %d",
2175                  oldEntry.getName(), newDN, oldID.longValue(),
2176                  newID.longValue());
2177            }
2178          }
2179
2180          // Move this entry.
2181          removeSubordinateEntry(txn, buffer, oldSuperiorDN,
2182              oldID, newID, oldEntry, newDN, isApexEntryMoved,
2183              modifyDNOperation, current);
2184          current = current.next;
2185
2186          if(modifyDNOperation != null)
2187          {
2188            modifyDNOperation.checkIfCanceled(false);
2189          }
2190
2191          // Get the next DN.
2192          data = new DatabaseEntry();
2193          status = cursor.getNext(key, data, LockMode.DEFAULT);
2194        }
2195      }
2196      finally
2197      {
2198        cursor.close();
2199      }
2200
2201      // Set current to the first moved entry and null out the head. This will
2202      // allow processed moved entries to be GCed.
2203      current = head.next;
2204      head = null;
2205      while(current != null)
2206      {
2207        addRenamedEntry(txn, buffer, current.entryID, current.entry,
2208                        isApexEntryMoved, current.renumbered,
2209                        modifyDNOperation);
2210        current = current.next;
2211      }
2212      buffer.flush(txn);
2213
2214      if(modifyDNOperation != null)
2215      {
2216        // One last check before committing
2217        modifyDNOperation.checkIfCanceled(true);
2218      }
2219
2220      // Commit the transaction.
2221      EntryContainer.transactionCommit(txn);
2222    }
2223    catch (DatabaseException | DirectoryException | CanceledOperationException e)
2224    {
2225      EntryContainer.transactionAbort(txn);
2226      throw e;
2227    }
2228    catch (Exception e)
2229    {
2230      EntryContainer.transactionAbort(txn);
2231
2232      String msg = e.getMessage();
2233      if (msg == null)
2234      {
2235        msg = stackTraceToSingleLineString(e);
2236      }
2237      LocalizableMessage message = ERR_UNCHECKED_EXCEPTION.get(msg);
2238      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
2239          message, e);
2240    }
2241  }
2242
2243  /** Represents an renamed entry that was deleted from JE but yet to be added back. */
2244  private static final class MovedEntry
2245  {
2246    private EntryID entryID;
2247    private Entry entry;
2248    private MovedEntry next;
2249    private boolean renumbered;
2250
2251    private MovedEntry(EntryID entryID, Entry entry, boolean renumbered)
2252    {
2253      this.entryID = entryID;
2254      this.entry = entry;
2255      this.renumbered = renumbered;
2256    }
2257  }
2258
2259  private void addRenamedEntry(Transaction txn, IndexBuffer buffer,
2260                           EntryID newID,
2261                           Entry newEntry,
2262                           boolean isApexEntryMoved,
2263                           boolean renumbered,
2264                           ModifyDNOperation modifyDNOperation)
2265      throws DirectoryException, DatabaseException
2266  {
2267    if (!dn2id.insert(txn, newEntry.getName(), newID))
2268    {
2269      throw new DirectoryException(
2270          ResultCode.ENTRY_ALREADY_EXISTS, ERR_MODIFYDN_ALREADY_EXISTS.get(newEntry.getName()));
2271    }
2272    id2entry.put(txn, newID, newEntry);
2273    dn2uri.addEntry(txn, newEntry);
2274
2275    if (renumbered || modifyDNOperation == null)
2276    {
2277      // Reindex the entry with the new ID.
2278      indexInsertEntry(buffer, newEntry, newID);
2279    }
2280
2281    // Add the new ID to id2children and id2subtree of new apex parent entry.
2282    if(isApexEntryMoved)
2283    {
2284      boolean isParent = true;
2285      for (DN dn = getParentWithinBase(newEntry.getName()); dn != null;
2286           dn = getParentWithinBase(dn))
2287      {
2288        EntryID parentID = dn2id.get(txn, dn, LockMode.DEFAULT);
2289        ByteString parentIDKeyBytes = ByteString.valueOf(parentID.longValue());
2290        if(isParent)
2291        {
2292          id2children.insertID(buffer, parentIDKeyBytes, newID);
2293          isParent = false;
2294        }
2295        id2subtree.insertID(buffer, parentIDKeyBytes, newID);
2296      }
2297    }
2298  }
2299
2300  private void removeApexEntry(Transaction txn, IndexBuffer buffer,
2301      DN oldSuperiorDN,
2302      EntryID oldID, EntryID newID,
2303      Entry oldEntry, Entry newEntry,
2304      boolean isApexEntryMoved,
2305      ModifyDNOperation modifyDNOperation,
2306      MovedEntry tail)
2307  throws DirectoryException, DatabaseException
2308  {
2309    DN oldDN = oldEntry.getName();
2310
2311    // Remove the old DN from dn2id.
2312    dn2id.remove(txn, oldDN);
2313
2314    // Remove old ID from id2entry and put the new entry
2315    // (old entry with new DN) in id2entry.
2316    if (!newID.equals(oldID))
2317    {
2318      id2entry.remove(txn, oldID);
2319    }
2320
2321    // Update any referral records.
2322    dn2uri.deleteEntry(txn, oldEntry);
2323
2324    tail.next = new MovedEntry(newID, newEntry, !newID.equals(oldID));
2325
2326    // Remove the old ID from id2children and id2subtree of
2327    // the old apex parent entry.
2328    if(oldSuperiorDN != null && isApexEntryMoved)
2329    {
2330      boolean isParent = true;
2331      for (DN dn = oldSuperiorDN; dn != null; dn = getParentWithinBase(dn))
2332      {
2333        EntryID parentID = dn2id.get(txn, dn, LockMode.DEFAULT);
2334        ByteString parentIDKeyBytes = ByteString.valueOf(parentID.longValue());
2335        if(isParent)
2336        {
2337          id2children.removeID(buffer, parentIDKeyBytes, oldID);
2338          isParent = false;
2339        }
2340        id2subtree.removeID(buffer, parentIDKeyBytes, oldID);
2341      }
2342    }
2343
2344    if (!newID.equals(oldID) || modifyDNOperation == null)
2345    {
2346      // All the subordinates will be renumbered so we have to rebuild
2347      // id2c and id2s with the new ID.
2348      ByteString oldIDKeyBytes = ByteString.valueOf(oldID.longValue());
2349      id2children.delete(buffer, oldIDKeyBytes);
2350      id2subtree.delete(buffer, oldIDKeyBytes);
2351
2352      // Reindex the entry with the new ID.
2353      indexRemoveEntry(buffer, oldEntry, oldID);
2354    }
2355    else
2356    {
2357      // Update the indexes if needed.
2358      indexModifications(buffer, oldEntry, newEntry, oldID,
2359          modifyDNOperation.getModifications());
2360    }
2361
2362    // Remove the entry from the entry cache.
2363    EntryCache<?> entryCache = DirectoryServer.getEntryCache();
2364    if (entryCache != null)
2365    {
2366      entryCache.removeEntry(oldDN);
2367    }
2368  }
2369
2370  private void removeSubordinateEntry(Transaction txn, IndexBuffer buffer,
2371      DN oldSuperiorDN,
2372      EntryID oldID, EntryID newID,
2373      Entry oldEntry, DN newDN,
2374      boolean isApexEntryMoved,
2375      ModifyDNOperation modifyDNOperation,
2376      MovedEntry tail)
2377  throws DirectoryException, DatabaseException
2378  {
2379    DN oldDN = oldEntry.getName();
2380    Entry newEntry = oldEntry.duplicate(false);
2381    newEntry.setDN(newDN);
2382    List<Modification> modifications =
2383      Collections.unmodifiableList(new ArrayList<Modification>(0));
2384
2385    // Create a new entry that is a copy of the old entry but with the new DN.
2386    // Also invoke any subordinate modify DN plugins on the entry.
2387    // FIXME -- At the present time, we don't support subordinate modify DN
2388    //          plugins that make changes to subordinate entries and therefore
2389    //          provide an unmodifiable list for the modifications element.
2390    // FIXME -- This will need to be updated appropriately if we decided that
2391    //          these plugins should be invoked for synchronization
2392    //          operations.
2393    if (modifyDNOperation != null && !modifyDNOperation.isSynchronizationOperation())
2394    {
2395      SubordinateModifyDN pluginResult =
2396        getPluginConfigManager().invokeSubordinateModifyDNPlugins(
2397            modifyDNOperation, oldEntry, newEntry, modifications);
2398
2399      if (!pluginResult.continueProcessing())
2400      {
2401        throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
2402            ERR_MODIFYDN_ABORTED_BY_SUBORDINATE_PLUGIN.get(oldDN, newDN));
2403      }
2404
2405      if (! modifications.isEmpty())
2406      {
2407        LocalizableMessageBuilder invalidReason = new LocalizableMessageBuilder();
2408        if (! newEntry.conformsToSchema(null, false, false, false,
2409            invalidReason))
2410        {
2411          throw new DirectoryException(
2412              DirectoryServer.getServerErrorResultCode(),
2413              ERR_MODIFYDN_ABORTED_BY_SUBORDINATE_SCHEMA_ERROR.get(oldDN, newDN, invalidReason));
2414        }
2415      }
2416    }
2417
2418    // Remove the old DN from dn2id.
2419    dn2id.remove(txn, oldDN);
2420
2421    // Remove old ID from id2entry and put the new entry
2422    // (old entry with new DN) in id2entry.
2423    if (!newID.equals(oldID))
2424    {
2425      id2entry.remove(txn, oldID);
2426    }
2427
2428    // Update any referral records.
2429    dn2uri.deleteEntry(txn, oldEntry);
2430
2431    tail.next = new MovedEntry(newID, newEntry, !newID.equals(oldID));
2432
2433    if(isApexEntryMoved)
2434    {
2435      // Remove the old ID from id2subtree of old apex superior entries.
2436      for (DN dn = oldSuperiorDN; dn != null; dn = getParentWithinBase(dn))
2437      {
2438        EntryID parentID = dn2id.get(txn, dn, LockMode.DEFAULT);
2439        ByteString parentIDKeyBytes = ByteString.valueOf(parentID.longValue());
2440        id2subtree.removeID(buffer, parentIDKeyBytes, oldID);
2441      }
2442    }
2443
2444    if (!newID.equals(oldID))
2445    {
2446      // All the subordinates will be renumbered so we have to rebuild
2447      // id2c and id2s with the new ID.
2448      ByteString oldIDKeyBytes = ByteString.valueOf(oldID.longValue());
2449      id2children.delete(buffer, oldIDKeyBytes);
2450      id2subtree.delete(buffer, oldIDKeyBytes);
2451
2452      // Reindex the entry with the new ID.
2453      indexRemoveEntry(buffer, oldEntry, oldID);
2454    }
2455    else if (!modifications.isEmpty())
2456    {
2457      // Update the indexes.
2458      indexModifications(buffer, oldEntry, newEntry, oldID, modifications);
2459    }
2460
2461    // Remove the entry from the entry cache.
2462    EntryCache<?> entryCache = DirectoryServer.getEntryCache();
2463    if (entryCache != null)
2464    {
2465      entryCache.removeEntry(oldDN);
2466    }
2467  }
2468
2469  /**
2470   * Make a new DN for a subordinate entry of a renamed or moved entry.
2471   *
2472   * @param oldDN The current DN of the subordinate entry.
2473   * @param oldSuffixLen The current DN length of the renamed or moved entry.
2474   * @param newSuffixDN The new DN of the renamed or moved entry.
2475   * @return The new DN of the subordinate entry.
2476   */
2477  static DN modDN(DN oldDN, int oldSuffixLen, DN newSuffixDN)
2478  {
2479    int oldDNNumComponents    = oldDN.size();
2480    int oldDNKeepComponents   = oldDNNumComponents - oldSuffixLen;
2481    int newSuffixDNComponents = newSuffixDN.size();
2482
2483    RDN[] newDNComponents = new RDN[oldDNKeepComponents+newSuffixDNComponents];
2484    for (int i=0; i < oldDNKeepComponents; i++)
2485    {
2486      newDNComponents[i] = oldDN.getRDN(i);
2487    }
2488
2489    for (int i=oldDNKeepComponents, j=0; j < newSuffixDNComponents; i++,j++)
2490    {
2491      newDNComponents[i] = newSuffixDN.getRDN(j);
2492    }
2493
2494    return new DN(newDNComponents);
2495  }
2496
2497  /**
2498   * Insert a new entry into the attribute indexes.
2499   *
2500   * @param buffer The index buffer used to buffer up the index changes.
2501   * @param entry The entry to be inserted into the indexes.
2502   * @param entryID The ID of the entry to be inserted into the indexes.
2503   * @throws DatabaseException If an error occurs in the JE database.
2504   * @throws DirectoryException If a Directory Server error occurs.
2505   */
2506  private void indexInsertEntry(IndexBuffer buffer, Entry entry, EntryID entryID)
2507      throws DatabaseException, DirectoryException
2508  {
2509    for (AttributeIndex index : attrIndexMap.values())
2510    {
2511      index.addEntry(buffer, entryID, entry);
2512    }
2513
2514    for (VLVIndex vlvIndex : vlvIndexMap.values())
2515    {
2516      vlvIndex.addEntry(buffer, entryID, entry);
2517    }
2518  }
2519
2520  /**
2521   * Remove an entry from the attribute indexes.
2522   *
2523   * @param buffer The index buffer used to buffer up the index changes.
2524   * @param entry The entry to be removed from the indexes.
2525   * @param entryID The ID of the entry to be removed from the indexes.
2526   * @throws DatabaseException If an error occurs in the JE database.
2527   * @throws DirectoryException If a Directory Server error occurs.
2528   */
2529  private void indexRemoveEntry(IndexBuffer buffer, Entry entry, EntryID entryID)
2530      throws DatabaseException, DirectoryException
2531  {
2532    for (AttributeIndex index : attrIndexMap.values())
2533    {
2534      index.removeEntry(buffer, entryID, entry);
2535    }
2536
2537    for (VLVIndex vlvIndex : vlvIndexMap.values())
2538    {
2539      vlvIndex.removeEntry(buffer, entryID, entry);
2540    }
2541  }
2542
2543  /**
2544   * Update the attribute indexes to reflect the changes to the
2545   * attributes of an entry resulting from a sequence of modifications.
2546   *
2547   * @param buffer The index buffer used to buffer up the index changes.
2548   * @param oldEntry The contents of the entry before the change.
2549   * @param newEntry The contents of the entry after the change.
2550   * @param entryID The ID of the entry that was changed.
2551   * @param mods The sequence of modifications made to the entry.
2552   * @throws DatabaseException If an error occurs in the JE database.
2553   * @throws DirectoryException If a Directory Server error occurs.
2554   */
2555  private void indexModifications(IndexBuffer buffer, Entry oldEntry, Entry newEntry,
2556      EntryID entryID, List<Modification> mods)
2557  throws DatabaseException, DirectoryException
2558  {
2559    // Process in index configuration order.
2560    for (AttributeIndex index : attrIndexMap.values())
2561    {
2562      // Check whether any modifications apply to this indexed attribute.
2563      if (isAttributeModified(index, mods))
2564      {
2565        index.modifyEntry(buffer, entryID, oldEntry, newEntry, mods);
2566      }
2567    }
2568
2569    for(VLVIndex vlvIndex : vlvIndexMap.values())
2570    {
2571      vlvIndex.modifyEntry(buffer, entryID, oldEntry, newEntry, mods);
2572    }
2573  }
2574
2575  /**
2576   * Get a count of the number of entries stored in this entry container.
2577   *
2578   * @return The number of entries stored in this entry container.
2579   * @throws DatabaseException If an error occurs in the JE database.
2580   */
2581  public long getEntryCount() throws DatabaseException
2582  {
2583    EntryID entryID = dn2id.get(null, baseDN, LockMode.DEFAULT);
2584    if (entryID != null)
2585    {
2586      DatabaseEntry key = new DatabaseEntry(entryIDToDatabase(entryID.longValue()));
2587      EntryIDSet entryIDSet = id2subtree.readKey(key, null, LockMode.DEFAULT);
2588
2589      long count = entryIDSet.size();
2590      if(count != Long.MAX_VALUE)
2591      {
2592        // Add the base entry itself
2593        return ++count;
2594      }
2595      else
2596      {
2597        // The count is not maintained. Fall back to the slow method
2598        return id2entry.getRecordCount();
2599      }
2600    }
2601    else
2602    {
2603      // Base entry doesn't not exist so this entry container
2604      // must not have any entries
2605      return 0;
2606    }
2607  }
2608
2609  /**
2610   * Get the number of values for which the entry limit has been exceeded
2611   * since the entry container was opened.
2612   * @return The number of values for which the entry limit has been exceeded.
2613   */
2614  public int getEntryLimitExceededCount()
2615  {
2616    int count = 0;
2617    count += id2children.getEntryLimitExceededCount();
2618    count += id2subtree.getEntryLimitExceededCount();
2619    for (AttributeIndex index : attrIndexMap.values())
2620    {
2621      count += index.getEntryLimitExceededCount();
2622    }
2623    return count;
2624  }
2625
2626
2627  /**
2628   * Get a list of the databases opened by the entryContainer.
2629   * @param dbList A list of database containers.
2630   */
2631  public void listDatabases(List<DatabaseContainer> dbList)
2632  {
2633    dbList.add(dn2id);
2634    dbList.add(id2entry);
2635    dbList.add(dn2uri);
2636    if (config.isSubordinateIndexesEnabled())
2637    {
2638      dbList.add(id2children);
2639      dbList.add(id2subtree);
2640    }
2641    dbList.add(state);
2642
2643    for(AttributeIndex index : attrIndexMap.values())
2644    {
2645      index.listDatabases(dbList);
2646    }
2647
2648    dbList.addAll(vlvIndexMap.values());
2649  }
2650
2651  /**
2652   * Determine whether the provided operation has the ManageDsaIT request
2653   * control.
2654   * @param operation The operation for which the determination is to be made.
2655   * @return true if the operation has the ManageDsaIT request control, or false
2656   * if not.
2657   */
2658  private static boolean isManageDsaITOperation(Operation operation)
2659  {
2660    if(operation != null)
2661    {
2662      List<Control> controls = operation.getRequestControls();
2663      if (controls != null)
2664      {
2665        for (Control control : controls)
2666        {
2667          if (ServerConstants.OID_MANAGE_DSAIT_CONTROL.equals(control.getOID()))
2668          {
2669            return true;
2670          }
2671        }
2672      }
2673    }
2674    return false;
2675  }
2676
2677  /**
2678   * Begin a leaf transaction using the default configuration.
2679   * Provides assertion debug logging.
2680   * @return A JE transaction handle.
2681   * @throws DatabaseException If an error occurs while attempting to begin
2682   * a new transaction.
2683   */
2684  public Transaction beginTransaction()
2685  throws DatabaseException
2686  {
2687    Transaction parentTxn = null;
2688    TransactionConfig txnConfig = null;
2689    Transaction txn = env.beginTransaction(parentTxn, txnConfig);
2690    if (logger.isTraceEnabled())
2691    {
2692      logger.trace("beginTransaction", "begin txnid=" + txn.getId());
2693    }
2694    return txn;
2695  }
2696
2697  /**
2698   * Commit a transaction.
2699   * Provides assertion debug logging.
2700   * @param txn The JE transaction handle.
2701   * @throws DatabaseException If an error occurs while attempting to commit
2702   * the transaction.
2703   */
2704  public static void transactionCommit(Transaction txn)
2705  throws DatabaseException
2706  {
2707    if (txn != null)
2708    {
2709      txn.commit();
2710      if (logger.isTraceEnabled())
2711      {
2712        logger.trace("commit txnid=%d", txn.getId());
2713      }
2714    }
2715  }
2716
2717  /**
2718   * Abort a transaction.
2719   * Provides assertion debug logging.
2720   * @param txn The JE transaction handle.
2721   * @throws DatabaseException If an error occurs while attempting to abort the
2722   * transaction.
2723   */
2724  public static void transactionAbort(Transaction txn)
2725  throws DatabaseException
2726  {
2727    if (txn != null)
2728    {
2729      txn.abort();
2730      if (logger.isTraceEnabled())
2731      {
2732        logger.trace("abort txnid=%d", txn.getId());
2733      }
2734    }
2735  }
2736
2737  /**
2738   * Delete this entry container from disk. The entry container should be
2739   * closed before calling this method.
2740   *
2741   * @throws DatabaseException If an error occurs while removing the entry
2742   *                           container.
2743   */
2744  void delete() throws DatabaseException
2745  {
2746    List<DatabaseContainer> databases = new ArrayList<>();
2747    listDatabases(databases);
2748
2749    if(env.getConfig().getTransactional())
2750    {
2751      Transaction txn = beginTransaction();
2752
2753      try
2754      {
2755        for(DatabaseContainer db : databases)
2756        {
2757          env.removeDatabase(txn, db.getName());
2758        }
2759
2760        transactionCommit(txn);
2761      }
2762      catch(DatabaseException de)
2763      {
2764        transactionAbort(txn);
2765        throw de;
2766      }
2767    }
2768    else
2769    {
2770      for(DatabaseContainer db : databases)
2771      {
2772        env.removeDatabase(null, db.getName());
2773      }
2774    }
2775  }
2776
2777  /**
2778   * Remove a database from disk.
2779   *
2780   * @param database The database container to remove.
2781   * @throws DatabaseException If an error occurs while attempting to delete the
2782   * database.
2783   */
2784  void deleteDatabase(DatabaseContainer database)
2785  throws DatabaseException
2786  {
2787    if(database == state)
2788    {
2789      // The state database can not be removed individually.
2790      return;
2791    }
2792
2793    database.close();
2794    if(env.getConfig().getTransactional())
2795    {
2796      Transaction txn = beginTransaction();
2797      try
2798      {
2799        env.removeDatabase(txn, database.getName());
2800        if(database instanceof Index)
2801        {
2802          state.removeIndexTrustState(txn, database);
2803        }
2804        transactionCommit(txn);
2805      }
2806      catch(DatabaseException de)
2807      {
2808        transactionAbort(txn);
2809        throw de;
2810      }
2811    }
2812    else
2813    {
2814      env.removeDatabase(null, database.getName());
2815      if(database instanceof Index)
2816      {
2817        state.removeIndexTrustState(null, database);
2818      }
2819    }
2820  }
2821
2822  /**
2823   * Removes a attribute index from disk.
2824   *
2825   * @param attributeIndex The attribute index to remove.
2826   * @throws DatabaseException If an JE database error occurs while attempting
2827   * to delete the index.
2828   */
2829  private void deleteAttributeIndex(AttributeIndex attributeIndex)
2830      throws DatabaseException
2831  {
2832    attributeIndex.close();
2833    Transaction txn = env.getConfig().getTransactional()
2834      ? beginTransaction() : null;
2835    try
2836    {
2837      for (Index index : attributeIndex.getAllIndexes())
2838      {
2839        env.removeDatabase(txn, index.getName());
2840        state.removeIndexTrustState(txn, index);
2841      }
2842      if (txn != null)
2843      {
2844        transactionCommit(txn);
2845      }
2846    }
2847    catch(DatabaseException de)
2848    {
2849      if (txn != null)
2850      {
2851        transactionAbort(txn);
2852      }
2853      throw de;
2854    }
2855  }
2856
2857  /**
2858   * This method constructs a container name from a base DN. Only alphanumeric
2859   * characters are preserved, all other characters are replaced with an
2860   * underscore.
2861   *
2862   * @return The container name for the base DN.
2863   */
2864  public String getDatabasePrefix()
2865  {
2866    return databasePrefix;
2867  }
2868
2869  /**
2870   * Sets a new database prefix for this entry container and rename all
2871   * existing databases in use by this entry container.
2872   *
2873   * @param newDatabasePrefix The new database prefix to use.
2874   * @throws DatabaseException If an error occurs in the JE database.
2875   * @throws JebException If an error occurs in the JE backend.
2876   */
2877  public void setDatabasePrefix(String newDatabasePrefix)
2878  throws DatabaseException, JebException
2879
2880  {
2881    List<DatabaseContainer> databases = new ArrayList<>();
2882    listDatabases(databases);
2883
2884    newDatabasePrefix = preparePrefix(newDatabasePrefix);
2885
2886    // close the containers.
2887    for(DatabaseContainer db : databases)
2888    {
2889      db.close();
2890    }
2891
2892    try
2893    {
2894      if(env.getConfig().getTransactional())
2895      {
2896        //Rename under transaction
2897        Transaction txn = beginTransaction();
2898        try
2899        {
2900          for(DatabaseContainer db : databases)
2901          {
2902            String oldName = db.getName();
2903            String newName = oldName.replace(databasePrefix, newDatabasePrefix);
2904            env.renameDatabase(txn, oldName, newName);
2905          }
2906
2907          transactionCommit(txn);
2908
2909          for(DatabaseContainer db : databases)
2910          {
2911            String oldName = db.getName();
2912            String newName = oldName.replace(databasePrefix, newDatabasePrefix);
2913            db.setName(newName);
2914          }
2915
2916          // Update the prefix.
2917          this.databasePrefix = newDatabasePrefix;
2918        }
2919        catch(Exception e)
2920        {
2921          transactionAbort(txn);
2922
2923          String msg = e.getMessage();
2924          if (msg == null)
2925          {
2926            msg = stackTraceToSingleLineString(e);
2927          }
2928          throw new JebException(ERR_UNCHECKED_EXCEPTION.get(msg), e);
2929        }
2930      }
2931      else
2932      {
2933        for(DatabaseContainer db : databases)
2934        {
2935          String oldName = db.getName();
2936          String newName = oldName.replace(databasePrefix, newDatabasePrefix);
2937          env.renameDatabase(null, oldName, newName);
2938          db.setName(newName);
2939        }
2940
2941        // Update the prefix.
2942        this.databasePrefix = newDatabasePrefix;
2943      }
2944    }
2945    finally
2946    {
2947      // Open the containers backup.
2948      for(DatabaseContainer db : databases)
2949      {
2950        db.open();
2951      }
2952    }
2953  }
2954
2955  /** {@inheritDoc} */
2956  @Override
2957  public DN getBaseDN()
2958  {
2959    return baseDN;
2960  }
2961
2962  /**
2963   * Get the parent of a DN in the scope of the base DN.
2964   *
2965   * @param dn A DN which is in the scope of the base DN.
2966   * @return The parent DN, or null if the given DN is the base DN.
2967   */
2968  DN getParentWithinBase(DN dn)
2969  {
2970    if (dn.equals(baseDN))
2971    {
2972      return null;
2973    }
2974    return dn.parent();
2975  }
2976
2977  /** {@inheritDoc} */
2978  @Override
2979  public boolean isConfigurationChangeAcceptable(
2980      LocalDBBackendCfg cfg, List<LocalizableMessage> unacceptableReasons)
2981  {
2982    // This is always true because only all config attributes used
2983    // by the entry container should be validated by the admin framework.
2984    return true;
2985  }
2986
2987  /** {@inheritDoc} */
2988  @Override
2989  public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg cfg)
2990  {
2991    final ConfigChangeResult ccr = new ConfigChangeResult();
2992
2993    exclusiveLock.lock();
2994    try
2995    {
2996      if (config.isSubordinateIndexesEnabled() != cfg.isSubordinateIndexesEnabled())
2997      {
2998        if (cfg.isSubordinateIndexesEnabled())
2999        {
3000          // Re-enabling subordinate indexes.
3001          openSubordinateIndexes();
3002        }
3003        else
3004        {
3005          // Disabling subordinate indexes. Use a null index and ensure that
3006          // future attempts to use the real indexes will fail.
3007          id2children.close();
3008          id2children = new NullIndex(databasePrefix + "_"
3009              + ID2CHILDREN_DATABASE_NAME, new ID2CIndexer(), state, env, this);
3010          state.putIndexTrustState(null, id2children, false);
3011          id2children.open(); // No-op
3012
3013          id2subtree.close();
3014          id2subtree = new NullIndex(databasePrefix + "_"
3015              + ID2SUBTREE_DATABASE_NAME, new ID2SIndexer(), state, env, this);
3016          state.putIndexTrustState(null, id2subtree, false);
3017          id2subtree.open(); // No-op
3018
3019          logger.info(NOTE_JEB_SUBORDINATE_INDEXES_DISABLED, cfg.getBackendId());
3020        }
3021      }
3022
3023      if (config.getIndexEntryLimit() != cfg.getIndexEntryLimit())
3024      {
3025        if (id2children.setIndexEntryLimit(cfg.getIndexEntryLimit()))
3026        {
3027          ccr.setAdminActionRequired(true);
3028          ccr.addMessage(NOTE_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(id2children.getName()));
3029        }
3030
3031        if (id2subtree.setIndexEntryLimit(cfg.getIndexEntryLimit()))
3032        {
3033          ccr.setAdminActionRequired(true);
3034          ccr.addMessage(NOTE_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(id2subtree.getName()));
3035        }
3036      }
3037
3038      DataConfig entryDataConfig = new DataConfig(cfg.isEntriesCompressed(),
3039          cfg.isCompactEncoding(), rootContainer.getCompressedSchema());
3040      id2entry.setDataConfig(entryDataConfig);
3041
3042      this.config = cfg;
3043    }
3044    catch (DatabaseException e)
3045    {
3046      ccr.addMessage(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
3047      ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
3048    }
3049    finally
3050    {
3051      exclusiveLock.unlock();
3052    }
3053
3054    return ccr;
3055  }
3056
3057  /**
3058   * Get the environment config of the JE environment used in this entry
3059   * container.
3060   *
3061   * @return The environment config of the JE environment.
3062   * @throws DatabaseException If an error occurs while retrieving the
3063   *                           configuration object.
3064   */
3065  public EnvironmentConfig getEnvironmentConfig() throws DatabaseException
3066  {
3067    return env.getConfig();
3068  }
3069
3070  /**
3071   * Clear the contents of this entry container.
3072   *
3073   * @throws DatabaseException If an error occurs while removing the entry
3074   *                           container.
3075   */
3076  public void clear() throws DatabaseException
3077  {
3078    List<DatabaseContainer> databases = new ArrayList<>();
3079    listDatabases(databases);
3080
3081    for(DatabaseContainer db : databases)
3082    {
3083      db.close();
3084    }
3085    try
3086    {
3087      if(env.getConfig().getTransactional())
3088      {
3089        Transaction txn = beginTransaction();
3090
3091        try
3092        {
3093          for(DatabaseContainer db : databases)
3094          {
3095            env.truncateDatabase(txn, db.getName(), false);
3096          }
3097
3098          transactionCommit(txn);
3099        }
3100        catch(DatabaseException de)
3101        {
3102          transactionAbort(txn);
3103          throw de;
3104        }
3105      }
3106      else
3107      {
3108        for(DatabaseContainer db : databases)
3109        {
3110          env.truncateDatabase(null, db.getName(), false);
3111        }
3112      }
3113    }
3114    finally
3115    {
3116      for(DatabaseContainer db : databases)
3117      {
3118        db.open();
3119      }
3120
3121      Transaction txn = null;
3122      try
3123      {
3124        if(env.getConfig().getTransactional()) {
3125          txn = beginTransaction();
3126        }
3127        for(DatabaseContainer db : databases)
3128        {
3129          if (db instanceof Index)
3130          {
3131            Index index = (Index)db;
3132            index.setTrusted(txn, true);
3133          }
3134        }
3135        if(env.getConfig().getTransactional()) {
3136          transactionCommit(txn);
3137        }
3138      }
3139      catch(Exception de)
3140      {
3141        logger.traceException(de);
3142
3143        // This is mainly used during the unit tests, so it's not essential.
3144        try
3145        {
3146          if (txn != null)
3147          {
3148            transactionAbort(txn);
3149          }
3150        }
3151        catch (Exception e)
3152        {
3153          logger.traceException(de);
3154        }
3155      }
3156    }
3157  }
3158
3159  /**
3160   * Clear the contents for a database from disk.
3161   *
3162   * @param database The database to clear.
3163   * @throws DatabaseException if a JE database error occurs.
3164   */
3165  public void clearDatabase(DatabaseContainer database)
3166  throws DatabaseException
3167  {
3168    database.close();
3169    try
3170    {
3171      if(env.getConfig().getTransactional())
3172      {
3173        Transaction txn = beginTransaction();
3174        try
3175        {
3176          env.removeDatabase(txn, database.getName());
3177          transactionCommit(txn);
3178        }
3179        catch(DatabaseException de)
3180        {
3181          transactionAbort(txn);
3182          throw de;
3183        }
3184      }
3185      else
3186      {
3187        env.removeDatabase(null, database.getName());
3188      }
3189    }
3190    finally
3191    {
3192      database.open();
3193    }
3194    if(logger.isTraceEnabled())
3195    {
3196      logger.trace("Cleared the database %s", database.getName());
3197    }
3198  }
3199
3200
3201  /**
3202   * Finds an existing entry whose DN is the closest ancestor of a given baseDN.
3203   *
3204   * @param baseDN  the DN for which we are searching a matched DN.
3205   * @return the DN of the closest ancestor of the baseDN.
3206   * @throws DirectoryException If an error prevented the check of an
3207   * existing entry from being performed.
3208   */
3209  private DN getMatchedDN(DN baseDN) throws DirectoryException
3210  {
3211    DN parentDN  = baseDN.getParentDNInSuffix();
3212    while (parentDN != null && parentDN.isDescendantOf(getBaseDN()))
3213    {
3214      if (entryExists(parentDN))
3215      {
3216        return parentDN;
3217      }
3218      parentDN = parentDN.getParentDNInSuffix();
3219    }
3220    return null;
3221  }
3222
3223  /**
3224   * Opens the id2children and id2subtree indexes.
3225   */
3226  private void openSubordinateIndexes()
3227  {
3228    id2children = newIndex(ID2CHILDREN_DATABASE_NAME, new ID2CIndexer());
3229    id2subtree = newIndex(ID2SUBTREE_DATABASE_NAME, new ID2SIndexer());
3230  }
3231
3232  private Index newIndex(String name, Indexer indexer)
3233  {
3234    final Index index = new Index(databasePrefix + "_" + name,
3235        indexer, state, config.getIndexEntryLimit(), 0, true, env, this);
3236    index.open();
3237    if (!index.isTrusted())
3238    {
3239      logger.info(NOTE_INDEX_ADD_REQUIRES_REBUILD, index.getName());
3240    }
3241    return index;
3242  }
3243
3244  /**
3245   * Creates a new index for an attribute.
3246   *
3247   * @param indexName the name to give to the new index
3248   * @param indexer the indexer to use when inserting data into the index
3249   * @param indexEntryLimit the index entry limit
3250   * @return a new index
3251   */
3252  Index newIndexForAttribute(String indexName, Indexer indexer, int indexEntryLimit)
3253  {
3254    final int cursorEntryLimit = 100000;
3255    return new Index(indexName, indexer, state, indexEntryLimit, cursorEntryLimit, false, env, this);
3256  }
3257
3258
3259  /**
3260   * Checks if any modifications apply to this indexed attribute.
3261   * @param index the indexed attributes.
3262   * @param mods the modifications to check for.
3263   * @return true if any apply, false otherwise.
3264   */
3265  private boolean isAttributeModified(AttributeIndex index,
3266                                      List<Modification> mods)
3267  {
3268    AttributeType indexAttributeType = index.getAttributeType();
3269    List<AttributeType> subTypes =
3270            DirectoryServer.getSchema().getSubTypes(indexAttributeType);
3271
3272    for (Modification mod : mods)
3273    {
3274      Attribute modAttr = mod.getAttribute();
3275      AttributeType modAttrType = modAttr.getAttributeType();
3276      if (modAttrType.equals(indexAttributeType)
3277          || subTypes.contains(modAttrType))
3278      {
3279        return true;
3280      }
3281    }
3282    return false;
3283  }
3284
3285
3286  /**
3287   * Fetch the base Entry of the EntryContainer.
3288   * @param baseDN the DN for the base entry
3289   * @param searchScope the scope under which this is fetched.
3290   *                    Scope is used for referral processing.
3291   * @return the Entry matching the baseDN.
3292   * @throws DirectoryException if the baseDN doesn't exist.
3293   */
3294  private Entry fetchBaseEntry(DN baseDN, SearchScope searchScope)
3295          throws DirectoryException
3296  {
3297    // Fetch the base entry.
3298    Entry baseEntry = null;
3299    try
3300    {
3301      baseEntry = getEntry(baseDN);
3302    }
3303    catch (Exception e)
3304    {
3305      logger.traceException(e);
3306    }
3307
3308    // The base entry must exist for a successful result.
3309    if (baseEntry == null)
3310    {
3311      // Check for referral entries above the base entry.
3312      dn2uri.targetEntryReferrals(baseDN, searchScope);
3313
3314      throw new DirectoryException(ResultCode.NO_SUCH_OBJECT,
3315            ERR_SEARCH_NO_SUCH_OBJECT.get(baseDN), getMatchedDN(baseDN), null);
3316    }
3317
3318    return baseEntry;
3319  }
3320
3321
3322  /**
3323   * Transform a database prefix string to one usable by the DB.
3324   * @param databasePrefix the database prefix
3325   * @return a new string when non letter or digit characters
3326   *         have been replaced with underscore
3327   */
3328  private String preparePrefix(String databasePrefix)
3329  {
3330    StringBuilder builder = new StringBuilder(databasePrefix.length());
3331    for (int i = 0; i < databasePrefix.length(); i++)
3332    {
3333      char ch = databasePrefix.charAt(i);
3334      if (Character.isLetterOrDigit(ch))
3335      {
3336        builder.append(ch);
3337      }
3338      else
3339      {
3340        builder.append('_');
3341      }
3342    }
3343    return builder.toString();
3344  }
3345
3346  /** Get the exclusive lock. */
3347  public void lock() {
3348    exclusiveLock.lock();
3349  }
3350
3351  /** Unlock the exclusive lock. */
3352  public void unlock() {
3353    exclusiveLock.unlock();
3354  }
3355
3356  /** {@inheritDoc} */
3357  @Override
3358  public String toString() {
3359    return databasePrefix;
3360  }
3361}