001/*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License").  You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
010 * or http://forgerock.org/license/CDDLv1.0.html.
011 * See the License for the specific language governing permissions
012 * and limitations under the License.
013 *
014 * When distributing Covered Code, include this CDDL HEADER in each
015 * file and include the License file at legal-notices/CDDLv1_0.txt.
016 * If applicable, add the following below this CDDL HEADER, with the
017 * fields enclosed by brackets "[]" replaced with your own identifying
018 * information:
019 *      Portions Copyright [yyyy] [name of copyright owner]
020 *
021 * CDDL HEADER END
022 *
023 *
024 *      Copyright 2006-2008 Sun Microsystems, Inc.
025 *      Portions Copyright 2012-2015 ForgeRock AS.
026 */
027package org.opends.server.backends.jeb;
028import com.sleepycat.je.Cursor;
029import com.sleepycat.je.CursorConfig;
030import com.sleepycat.je.DatabaseEntry;
031import com.sleepycat.je.DatabaseException;
032import com.sleepycat.je.LockMode;
033import com.sleepycat.je.OperationStatus;
034
035import org.opends.server.util.LDIFException;
036import org.opends.server.util.StaticUtils;
037
038import java.io.IOException;
039import java.util.*;
040
041import org.opends.server.types.*;
042import org.forgerock.opendj.ldap.ByteString;
043import org.forgerock.i18n.slf4j.LocalizedLogger;
044import static org.opends.messages.BackendMessages.*;
045
046/**
047 * Export a JE backend to LDIF.
048 */
049public class ExportJob
050{
051  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
052
053
054  /**
055   * The requested LDIF export configuration.
056   */
057  private LDIFExportConfig exportConfig;
058
059  /**
060   * The number of milliseconds between job progress reports.
061   */
062  private long progressInterval = 10000;
063
064  /**
065   * The current number of entries exported.
066   */
067  private long exportedCount;
068
069  /**
070   * The current number of entries skipped.
071   */
072  private long skippedCount;
073
074  /**
075   * Create a new export job.
076   *
077   * @param exportConfig The requested LDIF export configuration.
078   */
079  public ExportJob(LDIFExportConfig exportConfig)
080  {
081    this.exportConfig = exportConfig;
082  }
083
084  /**
085   * Export entries from the backend to an LDIF file.
086   * @param rootContainer The root container to export.
087   * @throws DatabaseException If an error occurs in the JE database.
088   * @throws IOException If an I/O error occurs while writing an entry.
089   * @throws JebException If an error occurs in the JE backend.
090   * @throws LDIFException If an error occurs while trying to determine whether
091   * to write an entry.
092   */
093  public void exportLDIF(RootContainer rootContainer)
094       throws IOException, LDIFException, DatabaseException, JebException
095  {
096    List<DN> includeBranches = exportConfig.getIncludeBranches();
097    ArrayList<EntryContainer> exportContainers = new ArrayList<>();
098
099    for (EntryContainer entryContainer : rootContainer.getEntryContainers())
100    {
101      // Skip containers that are not covered by the include branches.
102      DN baseDN = entryContainer.getBaseDN();
103
104      if (includeBranches == null || includeBranches.isEmpty())
105      {
106        exportContainers.add(entryContainer);
107      }
108      else
109      {
110        for (DN includeBranch : includeBranches)
111        {
112          if (includeBranch.isDescendantOf(baseDN) ||
113               includeBranch.isAncestorOf(baseDN))
114          {
115            exportContainers.add(entryContainer);
116            break;
117          }
118        }
119      }
120    }
121
122    // Make a note of the time we started.
123    long startTime = System.currentTimeMillis();
124
125    // Start a timer for the progress report.
126    Timer timer = new Timer();
127    TimerTask progressTask = new ProgressTask();
128    timer.scheduleAtFixedRate(progressTask, progressInterval,
129                              progressInterval);
130
131    // Iterate through the containers.
132    try
133    {
134      for (EntryContainer exportContainer : exportContainers)
135      {
136        if (exportConfig.isCancelled())
137        {
138          break;
139        }
140
141        exportContainer.sharedLock.lock();
142        try
143        {
144          exportContainer(exportContainer);
145        }
146        finally
147        {
148          exportContainer.sharedLock.unlock();
149        }
150      }
151    }
152    finally
153    {
154      timer.cancel();
155    }
156
157
158    long finishTime = System.currentTimeMillis();
159    long totalTime = finishTime - startTime;
160
161    float rate = 0;
162    if (totalTime > 0)
163    {
164      rate = 1000f*exportedCount / totalTime;
165    }
166
167    logger.info(NOTE_EXPORT_FINAL_STATUS, exportedCount, skippedCount, totalTime/1000, rate);
168
169  }
170
171  /**
172   * Export the entries in a single entry entryContainer, in other words from
173   * one of the base DNs.
174   * @param entryContainer The entry container that holds the entries to be
175   *                       exported.
176   * @throws DatabaseException If an error occurs in the JE database.
177   * @throws IOException If an error occurs while writing an entry.
178   * @throws  LDIFException  If an error occurs while trying to determine
179   *                         whether to write an entry.
180   */
181  private void exportContainer(EntryContainer entryContainer)
182       throws DatabaseException, IOException, LDIFException
183  {
184    ID2Entry id2entry = entryContainer.getID2Entry();
185
186    Cursor cursor = id2entry.openCursor(null, new CursorConfig());
187    try
188    {
189      DatabaseEntry key = new DatabaseEntry();
190      DatabaseEntry data = new DatabaseEntry();
191
192      OperationStatus status;
193      for (status = cursor.getFirst(key, data, LockMode.DEFAULT);
194           status == OperationStatus.SUCCESS;
195           status = cursor.getNext(key, data, LockMode.DEFAULT))
196      {
197        if (exportConfig.isCancelled())
198        {
199          break;
200        }
201
202        EntryID entryID = null;
203        try
204        {
205          entryID = new EntryID(key);
206        }
207        catch (Exception e)
208        {
209          if (logger.isTraceEnabled())
210          {
211            logger.traceException(e);
212
213            logger.trace("Malformed id2entry ID %s.%n",
214                            StaticUtils.bytesToHex(key.getData()));
215          }
216          skippedCount++;
217          continue;
218        }
219
220        if (entryID.longValue() == 0)
221        {
222          // This is the stored entry count.
223          continue;
224        }
225
226        Entry entry = null;
227        try
228        {
229          entry = ID2Entry.entryFromDatabase(ByteString.wrap(data.getData()),
230                       entryContainer.getRootContainer().getCompressedSchema());
231        }
232        catch (Exception e)
233        {
234          if (logger.isTraceEnabled())
235          {
236            logger.traceException(e);
237
238            logger.trace("Malformed id2entry record for ID %d:%n%s%n",
239                       entryID.longValue(),
240                       StaticUtils.bytesToHex(data.getData()));
241          }
242          skippedCount++;
243          continue;
244        }
245
246        if (entry.toLDIF(exportConfig))
247        {
248          exportedCount++;
249        }
250        else
251        {
252          skippedCount++;
253        }
254      }
255    }
256    finally
257    {
258      cursor.close();
259    }
260  }
261
262  /**
263   * This class reports progress of the export job at fixed intervals.
264   */
265  class ProgressTask extends TimerTask
266  {
267    /**
268     * The number of entries that had been exported at the time of the
269     * previous progress report.
270     */
271    private long previousCount;
272
273    /**
274     * The time in milliseconds of the previous progress report.
275     */
276    private long previousTime;
277
278    /**
279     * Create a new export progress task.
280     */
281    public ProgressTask()
282    {
283      previousTime = System.currentTimeMillis();
284    }
285
286    /**
287     * The action to be performed by this timer task.
288     */
289    public void run()
290    {
291      long latestCount = exportedCount;
292      long deltaCount = latestCount - previousCount;
293      long latestTime = System.currentTimeMillis();
294      long deltaTime = latestTime - previousTime;
295
296      if (deltaTime == 0)
297      {
298        return;
299      }
300
301      float rate = 1000f*deltaCount / deltaTime;
302
303      logger.info(NOTE_EXPORT_PROGRESS_REPORT, latestCount, skippedCount, rate);
304
305      previousCount = latestCount;
306      previousTime = latestTime;
307    }
308  }
309
310}