001/*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License").  You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
010 * or http://forgerock.org/license/CDDLv1.0.html.
011 * See the License for the specific language governing permissions
012 * and limitations under the License.
013 *
014 * When distributing Covered Code, include this CDDL HEADER in each
015 * file and include the License file at legal-notices/CDDLv1_0.txt.
016 * If applicable, add the following below this CDDL HEADER, with the
017 * fields enclosed by brackets "[]" replaced with your own identifying
018 * information:
019 *      Portions Copyright [yyyy] [name of copyright owner]
020 *
021 * CDDL HEADER END
022 *
023 *
024 *      Copyright 2006-2010 Sun Microsystems, Inc.
025 *      Portions Copyright 2011-2015 ForgeRock AS
026 *      Portions Copyright 2014 Manuel Gaupp
027 */
028package org.opends.server.backends.jeb;
029
030import static org.opends.messages.BackendMessages.*;
031import static org.opends.server.util.StaticUtils.*;
032
033import java.io.Closeable;
034import java.util.*;
035
036import org.forgerock.i18n.LocalizableMessage;
037import org.forgerock.i18n.LocalizableMessageBuilder;
038import org.forgerock.i18n.slf4j.LocalizedLogger;
039import org.forgerock.opendj.config.server.ConfigChangeResult;
040import org.forgerock.opendj.config.server.ConfigException;
041import org.forgerock.opendj.ldap.Assertion;
042import org.forgerock.opendj.ldap.ByteSequence;
043import org.forgerock.opendj.ldap.ByteString;
044import org.forgerock.opendj.ldap.DecodeException;
045import org.forgerock.opendj.ldap.schema.MatchingRule;
046import org.forgerock.opendj.ldap.spi.IndexQueryFactory;
047import org.forgerock.opendj.ldap.spi.Indexer;
048import org.forgerock.opendj.ldap.spi.IndexingOptions;
049import org.forgerock.util.Utils;
050import org.opends.server.admin.server.ConfigurationChangeListener;
051import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn.IndexType;
052import org.opends.server.admin.std.server.LocalDBIndexCfg;
053import org.opends.server.core.DirectoryServer;
054import org.opends.server.types.*;
055import org.opends.server.util.StaticUtils;
056
057import com.sleepycat.je.DatabaseException;
058
059/**
060 * Class representing an attribute index.
061 * We have a separate database for each type of indexing, which makes it easy
062 * to tell which attribute indexes are configured.  The different types of
063 * indexing are equality, presence, substrings and ordering.  The keys in the
064 * ordering index are ordered by setting the btree comparator to the ordering
065 * matching rule comparator.
066 * Note that the values in the equality index are normalized by the equality
067 * matching rule, whereas the values in the ordering index are normalized
068 * by the ordering matching rule.  If these could be guaranteed to be identical
069 * then we would not need a separate ordering index.
070 */
071public class AttributeIndex
072    implements ConfigurationChangeListener<LocalDBIndexCfg>, Closeable
073{
074  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
075
076  /** Type of the index filter. */
077  static enum IndexFilterType
078  {
079    /** Equality. */
080    EQUALITY(IndexType.EQUALITY),
081    /** Presence. */
082    PRESENCE(IndexType.PRESENCE),
083    /** Ordering. */
084    GREATER_OR_EQUAL(IndexType.ORDERING),
085    /** Ordering. */
086    LESS_OR_EQUAL(IndexType.ORDERING),
087    /** Substring. */
088    SUBSTRING(IndexType.SUBSTRING),
089    /** Approximate. */
090    APPROXIMATE(IndexType.APPROXIMATE);
091
092    private final IndexType indexType;
093
094    private IndexFilterType(IndexType indexType)
095    {
096      this.indexType = indexType;
097    }
098
099    /** {@inheritDoc} */
100    @Override
101    public String toString()
102    {
103      return indexType.toString();
104    }
105  }
106
107  /*
108   * FIXME Matthew Swift: Once the matching rules have been migrated we should
109   * revisit this class. All of the evaluateXXX methods should go (the Matcher
110   * class in the SDK could implement the logic, I hope).
111   */
112
113  /** The entryContainer in which this attribute index resides. */
114  private final EntryContainer entryContainer;
115
116  /** The attribute index configuration. */
117  private LocalDBIndexCfg indexConfig;
118  private IndexingOptions indexingOptions;
119
120  /** The mapping from names to indexes. */
121  private Map<String, Index> indexIdToIndexes;
122  private IndexQueryFactory<IndexQuery> indexQueryFactory;
123
124  /**
125   * Create a new attribute index object.
126   *
127   * @param indexConfig The attribute index configuration.
128   * @param entryContainer The entryContainer of this attribute index.
129   * @throws ConfigException if a configuration related error occurs.
130   */
131  public AttributeIndex(LocalDBIndexCfg indexConfig, EntryContainer entryContainer) throws ConfigException
132  {
133    this.entryContainer = entryContainer;
134    this.indexConfig = indexConfig;
135    this.indexingOptions = new JEIndexingOptions(indexConfig.getSubstringLength());
136    this.indexIdToIndexes = Collections.unmodifiableMap(buildIndexes(entryContainer, indexConfig, indexingOptions));
137    this.indexQueryFactory = new IndexQueryFactoryImpl(indexIdToIndexes, indexingOptions, indexConfig.getAttribute());
138  }
139
140  private static Map<String, Index> buildIndexes(EntryContainer entryContainer,
141                                                 LocalDBIndexCfg config,
142                                                 IndexingOptions options) throws ConfigException
143  {
144    final Map<String, Index> indexes = new HashMap<>();
145    final AttributeType attributeType = config.getAttribute();
146    final int indexEntryLimit = config.getIndexEntryLimit();
147
148    for(IndexType indexType : config.getIndexType()) {
149      Collection<? extends Indexer> indexers;
150      switch (indexType)
151      {
152      case PRESENCE:
153        indexes.put(indexType.toString(), newPresenceIndex(entryContainer, config));
154        indexers = Collections.emptyList();
155        break;
156      case EXTENSIBLE:
157        indexers = getExtensibleIndexers(config.getAttribute(), config.getIndexExtensibleMatchingRule(), options);
158        break;
159      case APPROXIMATE:
160        indexers =
161            throwIfNoMatchingRule(attributeType, indexType, attributeType.getApproximateMatchingRule())
162              .createIndexers(options);
163        break;
164      case EQUALITY:
165        indexers =
166            throwIfNoMatchingRule(attributeType, indexType, attributeType.getEqualityMatchingRule())
167              .createIndexers(options);
168        break;
169      case ORDERING:
170        indexers =
171            throwIfNoMatchingRule(attributeType, indexType, attributeType.getOrderingMatchingRule())
172              .createIndexers(options);
173        break;
174      case SUBSTRING:
175        indexers =
176            throwIfNoMatchingRule(attributeType, indexType, attributeType.getSubstringMatchingRule())
177              .createIndexers(options);
178        break;
179      default:
180       throw new ConfigException(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attributeType, indexType.toString()));
181      }
182      buildAndRegisterIndexesWithIndexers(entryContainer, attributeType, indexEntryLimit, indexers, indexes);
183    }
184
185    return indexes;
186  }
187
188  private static Index newPresenceIndex(EntryContainer entryContainer, LocalDBIndexCfg cfg)
189  {
190    final AttributeType attrType = cfg.getAttribute();
191    final String indexName = getIndexName(entryContainer, attrType, IndexType.PRESENCE.toString());
192    final PresenceIndexer indexer = new PresenceIndexer(attrType);
193    return entryContainer.newIndexForAttribute(indexName, indexer, cfg.getIndexEntryLimit());
194  }
195
196  private static MatchingRule throwIfNoMatchingRule(AttributeType attributeType, IndexType type, MatchingRule rule)
197      throws ConfigException
198  {
199    if (rule == null)
200    {
201      throw new ConfigException(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attributeType, type.toString()));
202    }
203    return rule;
204  }
205
206  private static void buildAndRegisterIndexesWithIndexers(EntryContainer entryContainer,
207                                                          AttributeType attributeType,
208                                                          int indexEntryLimit,
209                                                          Collection<? extends Indexer> indexers,
210                                                          Map<String, Index> indexes)
211  {
212    for (Indexer indexer : indexers)
213    {
214      final String indexID = indexer.getIndexID();
215      if (!indexes.containsKey(indexID))
216      {
217        final Index index = newAttributeIndex(entryContainer, attributeType, indexer, indexEntryLimit);
218        indexes.put(indexID, index);
219      }
220    }
221  }
222
223  private static Collection<Indexer> getExtensibleIndexers(AttributeType attributeType, Set<String> extensibleRules,
224      IndexingOptions options) throws ConfigException
225  {
226    if (extensibleRules == null || extensibleRules.isEmpty())
227    {
228      throw new ConfigException(
229          ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attributeType, IndexType.EXTENSIBLE.toString()));
230    }
231
232    final Collection<Indexer> indexers = new ArrayList<>();
233    for (final String ruleName : extensibleRules)
234    {
235      final MatchingRule rule = DirectoryServer.getMatchingRule(toLowerCase(ruleName));
236      if (rule == null)
237      {
238        logger.error(ERR_CONFIG_INDEX_TYPE_NEEDS_VALID_MATCHING_RULE, attributeType, ruleName);
239        continue;
240      }
241      indexers.addAll(rule.createIndexers(options));
242    }
243
244    return indexers;
245  }
246
247  private static MatchingRule getMatchingRule(IndexType indexType, AttributeType attrType)
248  {
249    switch (indexType)
250    {
251    case APPROXIMATE:
252      return attrType.getApproximateMatchingRule();
253    case EQUALITY:
254      return attrType.getEqualityMatchingRule();
255    case ORDERING:
256      return attrType.getOrderingMatchingRule();
257    case SUBSTRING:
258      return attrType.getSubstringMatchingRule();
259    default:
260      throw new IllegalArgumentException("Not implemented for index type " + indexType);
261    }
262  }
263
264  private static Index newAttributeIndex(EntryContainer entryContainer, AttributeType attributeType,
265      org.forgerock.opendj.ldap.spi.Indexer indexer, int indexEntryLimit)
266  {
267    final String indexName = getIndexName(entryContainer, attributeType, indexer.getIndexID());
268    final AttributeIndexer attrIndexer = new AttributeIndexer(attributeType, indexer);
269    return entryContainer.newIndexForAttribute(indexName, attrIndexer, indexEntryLimit);
270  }
271
272  private static String getIndexName(EntryContainer entryContainer, AttributeType attrType, String indexID)
273  {
274    return entryContainer.getDatabasePrefix() + "_" + attrType.getNameOrOID() + "." + indexID;
275  }
276
277  /**
278   * Open the attribute index.
279   *
280   * @throws DatabaseException if a JE database error occurs while
281   * opening the index.
282   */
283  public void open() throws DatabaseException
284  {
285    for (Index index : indexIdToIndexes.values())
286    {
287      index.open();
288    }
289    indexConfig.addChangeListener(this);
290  }
291
292  /** Closes the attribute index. */
293  @Override
294  public void close()
295  {
296    Utils.closeSilently(indexIdToIndexes.values());
297    indexConfig.removeChangeListener(this);
298    // The entryContainer is responsible for closing the JE databases.
299  }
300
301  /**
302   * Get the attribute type of this attribute index.
303   * @return The attribute type of this attribute index.
304   */
305  public AttributeType getAttributeType()
306  {
307    return indexConfig.getAttribute();
308  }
309
310  /**
311   * Return the indexing options of this AttributeIndex.
312   *
313   * @return the indexing options of this AttributeIndex.
314   */
315  public IndexingOptions getIndexingOptions()
316  {
317    return indexQueryFactory.getIndexingOptions();
318  }
319
320  /**
321   * Get the JE index configuration used by this index.
322   * @return The configuration in effect.
323   */
324  public LocalDBIndexCfg getConfiguration()
325  {
326    return indexConfig;
327  }
328
329  /**
330   * Update the attribute index for a new entry.
331   *
332   * @param buffer The index buffer to use to store the added keys
333   * @param entryID     The entry ID.
334   * @param entry       The contents of the new entry.
335   * @throws DatabaseException If an error occurs in the JE database.
336   * @throws DirectoryException If a Directory Server error occurs.
337   */
338  public void addEntry(IndexBuffer buffer, EntryID entryID, Entry entry) throws DatabaseException, DirectoryException
339  {
340    for (Index index : indexIdToIndexes.values())
341    {
342      index.addEntry(buffer, entryID, entry);
343    }
344  }
345
346  /**
347   * Update the attribute index for a deleted entry.
348   *
349   * @param buffer The index buffer to use to store the deleted keys
350   * @param entryID     The entry ID
351   * @param entry       The contents of the deleted entry.
352   * @throws DatabaseException If an error occurs in the JE database.
353   * @throws DirectoryException If a Directory Server error occurs.
354   */
355  public void removeEntry(IndexBuffer buffer, EntryID entryID, Entry entry)
356       throws DatabaseException, DirectoryException
357  {
358    for (Index index : indexIdToIndexes.values())
359    {
360      index.removeEntry(buffer, entryID, entry);
361    }
362  }
363
364  /**
365   * Update the index to reflect a sequence of modifications in a Modify
366   * operation.
367   *
368   * @param buffer The index buffer used to buffer up the index changes.
369   * @param entryID The ID of the entry that was modified.
370   * @param oldEntry The entry before the modifications were applied.
371   * @param newEntry The entry after the modifications were applied.
372   * @param mods The sequence of modifications in the Modify operation.
373   * @throws DatabaseException If an error occurs during an operation on a
374   * JE database.
375   */
376  public void modifyEntry(IndexBuffer buffer,
377                          EntryID entryID,
378                          Entry oldEntry,
379                          Entry newEntry,
380                          List<Modification> mods)
381       throws DatabaseException
382  {
383    for (Index index : indexIdToIndexes.values())
384    {
385      index.modifyEntry(buffer, entryID, oldEntry, newEntry, mods);
386    }
387  }
388
389  /**
390   * Makes a byte string representing a substring index key for
391   * one substring of a value.
392   *
393   * @param bytes The byte array containing the value.
394   * @param pos The starting position of the substring.
395   * @param len The length of the substring.
396   * @return A byte string containing a substring key.
397   */
398  private static ByteString makeSubstringKey(byte[] bytes, int pos, int len)
399  {
400    byte[] keyBytes = new byte[len];
401    System.arraycopy(bytes, pos, keyBytes, 0, len);
402    return ByteString.wrap(keyBytes);
403  }
404
405  /**
406   * Decompose an attribute value into a set of substring index keys.
407   * The ID of the entry containing this value should be inserted
408   * into the list of each of these keys.
409   *
410   * @param value A byte array containing the normalized attribute value.
411   * @return A set of index keys.
412   */
413  Set<ByteString> substringKeys(byte[] value)
414  { // FIXME replace this code with SDK's
415    // AbstractSubstringMatchingRuleImpl.SubstringIndexer.createKeys()
416
417    // Eliminate duplicates by putting the keys into a set.
418    // Sorting the keys will ensure database record locks are acquired
419    // in a consistent order and help prevent transaction deadlocks between
420    // concurrent writers.
421    Set<ByteString> set = new HashSet<>();
422
423    int substrLength = indexConfig.getSubstringLength();
424
425    // Example: The value is ABCDE and the substring length is 3.
426    // We produce the keys ABC BCD CDE DE E
427    // To find values containing a short substring such as DE,
428    // iterate through keys with prefix DE. To find values
429    // containing a longer substring such as BCDE, read keys BCD and CDE.
430    for (int i = 0, remain = value.length; remain > 0; i++, remain--)
431    {
432      int len = Math.min(substrLength, remain);
433      set.add(makeSubstringKey(value, i, len));
434    }
435    return set;
436  }
437
438  /**
439   * Retrieve the entry IDs that might match the provided assertion.
440   *
441   * @param indexQuery
442   *            The query used to retrieve entries.
443   * @param indexName
444   *            The name of index used to retrieve entries.
445   * @param filter
446   *          The filter on entries.
447   * @param debugBuffer
448   *          If not null, a diagnostic string will be written which will help
449   *          determine how the indexes contributed to this search.
450   * @param monitor
451   *          The database environment monitor provider that will keep index
452   *          filter usage statistics.
453   * @return The candidate entry IDs that might contain the filter assertion
454   *         value.
455   */
456  private EntryIDSet evaluateIndexQuery(IndexQuery indexQuery, String indexName, SearchFilter filter,
457      StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor)
458  {
459    LocalizableMessageBuilder debugMessage = monitor.isFilterUseEnabled() ? new LocalizableMessageBuilder() : null;
460    EntryIDSet results = indexQuery.evaluate(debugMessage);
461
462    if (debugBuffer != null)
463    {
464      debugBuffer.append("[INDEX:").append(indexConfig.getAttribute().getNameOrOID())
465        .append(".").append(indexName).append("]");
466    }
467
468    if (monitor.isFilterUseEnabled())
469    {
470      if (results.isDefined())
471      {
472        monitor.updateStats(filter, results.size());
473      }
474      else
475      {
476        monitor.updateStats(filter, debugMessage.toMessage());
477      }
478    }
479    return results;
480  }
481
482  /**
483   * Retrieve the entry IDs that might match two filters that restrict a value
484   * to both a lower bound and an upper bound.
485   *
486   * @param filter1
487   *          The first filter, that is either a less-or-equal filter or a
488   *          greater-or-equal filter.
489   * @param filter2
490   *          The second filter, that is either a less-or-equal filter or a
491   *          greater-or-equal filter. It must not be of the same type than the
492   *          first filter.
493   * @param debugBuffer
494   *          If not null, a diagnostic string will be written which will help
495   *          determine how the indexes contributed to this search.
496   * @param monitor
497   *          The database environment monitor provider that will keep index
498   *          filter usage statistics.
499   * @return The candidate entry IDs that might contain match both filters.
500   */
501  public EntryIDSet evaluateBoundedRange(SearchFilter filter1, SearchFilter filter2, StringBuilder debugBuffer,
502      DatabaseEnvironmentMonitor monitor)
503  {
504    // TODO : this implementation is not optimal
505    // as it implies two separate evaluations instead of a single one, thus defeating the purpose of
506    // the optimization done in IndexFilter#evaluateLogicalAndFilter method.
507    // One solution could be to implement a boundedRangeAssertion that combine the two operations in one.
508    // Such an optimization can only work for attributes declared as SINGLE-VALUE, though, since multiple
509    // values may match both filters with values outside the range. See OPENDJ-2194.
510    StringBuilder tmpBuff1 = debugBuffer != null ? new StringBuilder(): null;
511    StringBuilder tmpBuff2 = debugBuffer != null ? new StringBuilder(): null;
512    EntryIDSet results1 = evaluate(filter1, tmpBuff1, monitor);
513    EntryIDSet results2 = evaluate(filter2, tmpBuff2, monitor);
514    if (debugBuffer != null)
515    {
516      debugBuffer
517          .append(filter1).append(tmpBuff1).append(results1)
518          .append(filter2).append(tmpBuff2).append(results2);
519    }
520    results1.retainAll(results2);
521    return results1;
522  }
523
524  private EntryIDSet evaluate(SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor)
525  {
526    boolean isLessOrEqual = filter.getFilterType() == FilterType.LESS_OR_EQUAL;
527    IndexFilterType indexFilterType = isLessOrEqual ? IndexFilterType.LESS_OR_EQUAL : IndexFilterType.GREATER_OR_EQUAL;
528    return evaluateFilter(indexFilterType, filter, debugBuffer, monitor);
529  }
530
531  /**
532   * Retrieve the entry IDs that might match a filter.
533   *
534   * @param indexFilterType the index type filter
535   * @param filter The filter.
536   * @param debugBuffer If not null, a diagnostic string will be written
537   *                     which will help determine how the indexes contributed
538   *                     to this search.
539   * @param monitor The database environment monitor provider that will keep
540   *                index filter usage statistics.
541   * @return The candidate entry IDs that might contain a value
542   *         that matches the filter type.
543   */
544  public EntryIDSet evaluateFilter(IndexFilterType indexFilterType, SearchFilter filter, StringBuilder debugBuffer,
545      DatabaseEnvironmentMonitor monitor)
546  {
547    try
548    {
549      final IndexQuery indexQuery = getIndexQuery(indexFilterType, filter);
550      return evaluateIndexQuery(indexQuery, indexFilterType.toString(), filter, debugBuffer, monitor);
551    }
552    catch (DecodeException e)
553    {
554      logger.traceException(e);
555      return new EntryIDSet();
556    }
557  }
558
559  private IndexQuery getIndexQuery(IndexFilterType indexFilterType, SearchFilter filter) throws DecodeException
560  {
561    MatchingRule rule;
562    Assertion assertion;
563    switch (indexFilterType)
564    {
565    case EQUALITY:
566      rule = filter.getAttributeType().getEqualityMatchingRule();
567      assertion = rule.getAssertion(filter.getAssertionValue());
568      return assertion.createIndexQuery(indexQueryFactory);
569
570    case PRESENCE:
571      return indexQueryFactory.createMatchAllQuery();
572
573    case GREATER_OR_EQUAL:
574      rule = filter.getAttributeType().getOrderingMatchingRule();
575      assertion = rule.getGreaterOrEqualAssertion(filter.getAssertionValue());
576      return assertion.createIndexQuery(indexQueryFactory);
577
578    case LESS_OR_EQUAL:
579      rule = filter.getAttributeType().getOrderingMatchingRule();
580      assertion = rule.getLessOrEqualAssertion(filter.getAssertionValue());
581      return assertion.createIndexQuery(indexQueryFactory);
582
583    case SUBSTRING:
584      rule = filter.getAttributeType().getSubstringMatchingRule();
585      assertion = rule.getSubstringAssertion(
586          filter.getSubInitialElement(), filter.getSubAnyElements(), filter.getSubFinalElement());
587      return assertion.createIndexQuery(indexQueryFactory);
588
589    case APPROXIMATE:
590      rule = filter.getAttributeType().getApproximateMatchingRule();
591      assertion = rule.getAssertion(filter.getAssertionValue());
592      return assertion.createIndexQuery(indexQueryFactory);
593
594    default:
595      return null;
596    }
597  }
598
599  /**
600   * Delegator to {@link ByteSequence#BYTE_ARRAY_COMPARATOR}.
601   * <p>
602   * This intermediate class is necessary to satisfy JE's requirements for a btree comparator.
603   *
604   * @see com.sleepycat.je.DatabaseConfig#setBtreeComparator(Comparator)
605   */
606  public static class KeyComparator implements Comparator<byte[]>
607  {
608    /** The instance. */
609    public static final KeyComparator INSTANCE = new KeyComparator();
610
611    /** {@inheritDoc} */
612    @Override
613    public int compare(byte[] a, byte[] b)
614    {
615      return ByteSequence.BYTE_ARRAY_COMPARATOR.compare(a, b);
616    }
617  }
618
619  /**
620   * Return the number of values that have exceeded the entry limit since this
621   * object was created.
622   *
623   * @return The number of values that have exceeded the entry limit.
624   */
625  public long getEntryLimitExceededCount()
626  {
627    long entryLimitExceededCount = 0;
628
629    for (Index index : indexIdToIndexes.values())
630    {
631      entryLimitExceededCount += index.getEntryLimitExceededCount();
632    }
633    return entryLimitExceededCount;
634  }
635
636  /**
637   * Get a list of the databases opened by this attribute index.
638   * @param dbList A list of database containers.
639   */
640  public void listDatabases(List<DatabaseContainer> dbList)
641  {
642    dbList.addAll(indexIdToIndexes.values());
643  }
644
645  /**
646   * Get a string representation of this object.
647   * @return return A string representation of this object.
648   */
649  @Override
650  public String toString()
651  {
652    return getName();
653  }
654
655  /** {@inheritDoc} */
656  @Override
657  public synchronized boolean isConfigurationChangeAcceptable(
658      LocalDBIndexCfg cfg, List<LocalizableMessage> unacceptableReasons)
659  {
660    if (!isIndexAcceptable(cfg, IndexType.EQUALITY, unacceptableReasons)
661        || !isIndexAcceptable(cfg, IndexType.SUBSTRING, unacceptableReasons)
662        || !isIndexAcceptable(cfg, IndexType.ORDERING, unacceptableReasons)
663        || !isIndexAcceptable(cfg, IndexType.APPROXIMATE, unacceptableReasons))
664    {
665      return false;
666    }
667
668    AttributeType attrType = cfg.getAttribute();
669    if (cfg.getIndexType().contains(IndexType.EXTENSIBLE))
670    {
671      Set<String> newRules = cfg.getIndexExtensibleMatchingRule();
672      if (newRules == null || newRules.isEmpty())
673      {
674        unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "extensible"));
675        return false;
676      }
677    }
678    return true;
679  }
680
681  private static boolean isIndexAcceptable(LocalDBIndexCfg cfg, IndexType indexType,
682      List<LocalizableMessage> unacceptableReasons)
683  {
684    final AttributeType attrType = cfg.getAttribute();
685    if (cfg.getIndexType().contains(indexType)
686        && getMatchingRule(indexType, attrType) == null)
687    {
688      unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, indexType.toString()));
689      return false;
690    }
691    return true;
692  }
693
694  /** {@inheritDoc} */
695  @Override
696  public synchronized ConfigChangeResult applyConfigurationChange(final LocalDBIndexCfg newConfiguration)
697  {
698    final ConfigChangeResult ccr = new ConfigChangeResult();
699    final IndexingOptions newIndexingOptions = new JEIndexingOptions(newConfiguration.getSubstringLength());
700    try
701    {
702      Map<String, Index> newIndexIdToIndexes = buildIndexes(entryContainer, newConfiguration, newIndexingOptions);
703
704      final Map<String, Index> removedIndexes = new HashMap<>(indexIdToIndexes);
705      removedIndexes.keySet().removeAll(newIndexIdToIndexes.keySet());
706
707      final Map<String, Index> addedIndexes = new HashMap<>(newIndexIdToIndexes);
708      addedIndexes.keySet().removeAll(indexIdToIndexes.keySet());
709
710      final Map<String, Index> updatedIndexes = new HashMap<>(indexIdToIndexes);
711      updatedIndexes.keySet().retainAll(newIndexIdToIndexes.keySet());
712
713      // Replace instances of Index created by buildIndexes() with the one already opened and present in the actual
714      // indexIdToIndexes
715      newIndexIdToIndexes.putAll(updatedIndexes);
716
717      // Open added indexes *before* adding them to indexIdToIndexes
718      for (Index addedIndex : addedIndexes.values())
719      {
720        openIndex(addedIndex, ccr);
721      }
722
723      indexConfig = newConfiguration;
724      indexingOptions = newIndexingOptions;
725      indexIdToIndexes =  Collections.unmodifiableMap(newIndexIdToIndexes);
726      indexQueryFactory = new IndexQueryFactoryImpl(indexIdToIndexes, indexingOptions, indexConfig.getAttribute());
727
728      // FIXME: There is no guarantee here that deleted index are not currently involved in a query
729      for (Index removedIndex : removedIndexes.values())
730      {
731        deleteIndex(entryContainer, removedIndex);
732      }
733
734      for (Index updatedIndex : updatedIndexes.values())
735      {
736        updateIndex(updatedIndex, newConfiguration.getIndexEntryLimit(), ccr);
737      }
738    }
739    catch (Exception e)
740    {
741      ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
742      ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(e)));
743    }
744
745    return ccr;
746  }
747
748  private static void openIndex(Index index, ConfigChangeResult ccr)
749  {
750    index.open();
751    if (!index.isTrusted())
752    {
753      ccr.setAdminActionRequired(true);
754      ccr.addMessage(NOTE_INDEX_ADD_REQUIRES_REBUILD.get(index.getName()));
755    }
756  }
757
758  private static void updateIndex(Index updatedIndex, int newIndexEntryLimit, ConfigChangeResult ccr)
759  {
760    if (updatedIndex.setIndexEntryLimit(newIndexEntryLimit))
761    {
762      // This index can still be used since index size limit doesn't impact validity of the results.
763      ccr.setAdminActionRequired(true);
764      ccr.addMessage(NOTE_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(updatedIndex.getName()));
765    }
766  }
767
768  private static void deleteIndex(EntryContainer entryContainer, Index index)
769  {
770    entryContainer.exclusiveLock.lock();
771    try
772    {
773      entryContainer.deleteDatabase(index);
774    }
775    finally
776    {
777      entryContainer.exclusiveLock.unlock();
778    }
779  }
780
781  /**
782   * Return true iff this index is trusted.
783   * @return the trusted state of this index
784   */
785  public boolean isTrusted()
786  {
787    for (Index index : indexIdToIndexes.values())
788    {
789      if (!index.isTrusted())
790      {
791        return false;
792      }
793    }
794    return true;
795  }
796
797  /**
798   * Get the JE database name prefix for indexes in this attribute index.
799   *
800   * @return JE database name for this database container.
801   */
802  public String getName()
803  {
804    return entryContainer.getDatabasePrefix()
805        + "_"
806        + indexConfig.getAttribute().getNameOrOID();
807  }
808
809  Index getIndex(String indexID) {
810    return indexIdToIndexes.get(indexID);
811  }
812
813  /**
814   * Retrieves all the indexes used by this attribute index.
815   *
816   * @return An immutable collection of all indexes in use by this attribute
817   * index.
818   */
819  public Collection<Index> getAllIndexes() {
820    return indexIdToIndexes.values();
821  }
822
823  /**
824   * Retrieve the entry IDs that might match an extensible filter.
825   *
826   * @param filter The extensible filter.
827   * @param debugBuffer If not null, a diagnostic string will be written
828   *                     which will help determine how the indexes contributed
829   *                     to this search.
830   * @param monitor The database environment monitor provider that will keep
831   *                index filter usage statistics.
832   * @return The candidate entry IDs that might contain the filter
833   *         assertion value.
834   */
835  public EntryIDSet evaluateExtensibleFilter(SearchFilter filter,
836                                             StringBuilder debugBuffer,
837                                             DatabaseEnvironmentMonitor monitor)
838  {
839    //Get the Matching Rule OID of the filter.
840    String matchRuleOID  = filter.getMatchingRuleID();
841    /*
842     * Use the default equality index in two conditions:
843     * 1. There is no matching rule provided
844     * 2. The matching rule specified is actually the default equality.
845     */
846    MatchingRule eqRule = indexConfig.getAttribute().getEqualityMatchingRule();
847    if (matchRuleOID == null
848        || matchRuleOID.equals(eqRule.getOID())
849        || matchRuleOID.equalsIgnoreCase(eqRule.getNameOrOID()))
850    {
851      //No matching rule is defined; use the default equality matching rule.
852      return evaluateFilter(IndexFilterType.EQUALITY, filter, debugBuffer, monitor);
853    }
854
855    MatchingRule rule = DirectoryServer.getMatchingRule(matchRuleOID);
856    if (!ruleHasAtLeasOneIndex(rule))
857    {
858      if (monitor.isFilterUseEnabled())
859      {
860        monitor.updateStats(filter, INFO_INDEX_FILTER_MATCHING_RULE_NOT_INDEXED.get(
861            matchRuleOID, indexConfig.getAttribute().getNameOrOID()));
862      }
863      return IndexQuery.createNullIndexQuery().evaluate(null);
864    }
865
866    try
867    {
868      if (debugBuffer != null)
869      {
870        debugBuffer.append("[INDEX:");
871        for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.createIndexers(indexingOptions))
872        {
873            debugBuffer.append(" ")
874              .append(filter.getAttributeType().getNameOrOID())
875              .append(".")
876              .append(indexer.getIndexID());
877        }
878        debugBuffer.append("]");
879      }
880
881      final IndexQuery indexQuery = rule.getAssertion(filter.getAssertionValue()).createIndexQuery(indexQueryFactory);
882      LocalizableMessageBuilder debugMessage = monitor.isFilterUseEnabled() ? new LocalizableMessageBuilder() : null;
883      EntryIDSet results = indexQuery.evaluate(debugMessage);
884      if (monitor.isFilterUseEnabled())
885      {
886        if (results.isDefined())
887        {
888          monitor.updateStats(filter, results.size());
889        }
890        else
891        {
892          monitor.updateStats(filter, debugMessage.toMessage());
893        }
894      }
895      return results;
896    }
897    catch (DecodeException e)
898    {
899      logger.traceException(e);
900      return IndexQuery.createNullIndexQuery().evaluate(null);
901    }
902  }
903
904  private boolean ruleHasAtLeasOneIndex(MatchingRule rule)
905  {
906    for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.createIndexers(indexingOptions))
907    {
908      if (indexIdToIndexes.containsKey(indexer.getIndexID()))
909      {
910        return true;
911      }
912    }
913    return false;
914  }
915
916  /** This class extends the IndexConfig for JE Backend. */
917  private static final class JEIndexingOptions implements IndexingOptions
918  {
919    /** The length of the substring index. */
920    private int substringLength;
921
922    /**
923     * Creates a new JEIndexConfig instance.
924     * @param substringLength The length of the substring.
925     */
926    private JEIndexingOptions(int substringLength)
927    {
928      this.substringLength = substringLength;
929    }
930
931    /** {@inheritDoc} */
932    @Override
933    public int substringKeySize()
934    {
935      return substringLength;
936    }
937  }
938}