001/* 002 * CDDL HEADER START 003 * 004 * The contents of this file are subject to the terms of the 005 * Common Development and Distribution License, Version 1.0 only 006 * (the "License"). You may not use this file except in compliance 007 * with the License. 008 * 009 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt 010 * or http://forgerock.org/license/CDDLv1.0.html. 011 * See the License for the specific language governing permissions 012 * and limitations under the License. 013 * 014 * When distributing Covered Code, include this CDDL HEADER in each 015 * file and include the License file at legal-notices/CDDLv1_0.txt. 016 * If applicable, add the following below this CDDL HEADER, with the 017 * fields enclosed by brackets "[]" replaced with your own identifying 018 * information: 019 * Portions Copyright [yyyy] [name of copyright owner] 020 * 021 * CDDL HEADER END 022 * 023 * 024 * Copyright 2006-2008 Sun Microsystems, Inc. 025 * Portions Copyright 2012-2015 ForgeRock AS. 026 */ 027package org.opends.server.backends.jeb; 028import com.sleepycat.je.Cursor; 029import com.sleepycat.je.CursorConfig; 030import com.sleepycat.je.DatabaseEntry; 031import com.sleepycat.je.DatabaseException; 032import com.sleepycat.je.LockMode; 033import com.sleepycat.je.OperationStatus; 034 035import org.opends.server.util.LDIFException; 036import org.opends.server.util.StaticUtils; 037 038import java.io.IOException; 039import java.util.*; 040 041import org.opends.server.types.*; 042import org.forgerock.opendj.ldap.ByteString; 043import org.forgerock.i18n.slf4j.LocalizedLogger; 044import static org.opends.messages.BackendMessages.*; 045 046/** 047 * Export a JE backend to LDIF. 048 */ 049public class ExportJob 050{ 051 private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); 052 053 054 /** 055 * The requested LDIF export configuration. 056 */ 057 private LDIFExportConfig exportConfig; 058 059 /** 060 * The number of milliseconds between job progress reports. 061 */ 062 private long progressInterval = 10000; 063 064 /** 065 * The current number of entries exported. 066 */ 067 private long exportedCount; 068 069 /** 070 * The current number of entries skipped. 071 */ 072 private long skippedCount; 073 074 /** 075 * Create a new export job. 076 * 077 * @param exportConfig The requested LDIF export configuration. 078 */ 079 public ExportJob(LDIFExportConfig exportConfig) 080 { 081 this.exportConfig = exportConfig; 082 } 083 084 /** 085 * Export entries from the backend to an LDIF file. 086 * @param rootContainer The root container to export. 087 * @throws DatabaseException If an error occurs in the JE database. 088 * @throws IOException If an I/O error occurs while writing an entry. 089 * @throws JebException If an error occurs in the JE backend. 090 * @throws LDIFException If an error occurs while trying to determine whether 091 * to write an entry. 092 */ 093 public void exportLDIF(RootContainer rootContainer) 094 throws IOException, LDIFException, DatabaseException, JebException 095 { 096 List<DN> includeBranches = exportConfig.getIncludeBranches(); 097 ArrayList<EntryContainer> exportContainers = new ArrayList<>(); 098 099 for (EntryContainer entryContainer : rootContainer.getEntryContainers()) 100 { 101 // Skip containers that are not covered by the include branches. 102 DN baseDN = entryContainer.getBaseDN(); 103 104 if (includeBranches == null || includeBranches.isEmpty()) 105 { 106 exportContainers.add(entryContainer); 107 } 108 else 109 { 110 for (DN includeBranch : includeBranches) 111 { 112 if (includeBranch.isDescendantOf(baseDN) || 113 includeBranch.isAncestorOf(baseDN)) 114 { 115 exportContainers.add(entryContainer); 116 break; 117 } 118 } 119 } 120 } 121 122 // Make a note of the time we started. 123 long startTime = System.currentTimeMillis(); 124 125 // Start a timer for the progress report. 126 Timer timer = new Timer(); 127 TimerTask progressTask = new ProgressTask(); 128 timer.scheduleAtFixedRate(progressTask, progressInterval, 129 progressInterval); 130 131 // Iterate through the containers. 132 try 133 { 134 for (EntryContainer exportContainer : exportContainers) 135 { 136 if (exportConfig.isCancelled()) 137 { 138 break; 139 } 140 141 exportContainer.sharedLock.lock(); 142 try 143 { 144 exportContainer(exportContainer); 145 } 146 finally 147 { 148 exportContainer.sharedLock.unlock(); 149 } 150 } 151 } 152 finally 153 { 154 timer.cancel(); 155 } 156 157 158 long finishTime = System.currentTimeMillis(); 159 long totalTime = finishTime - startTime; 160 161 float rate = 0; 162 if (totalTime > 0) 163 { 164 rate = 1000f*exportedCount / totalTime; 165 } 166 167 logger.info(NOTE_EXPORT_FINAL_STATUS, exportedCount, skippedCount, totalTime/1000, rate); 168 169 } 170 171 /** 172 * Export the entries in a single entry entryContainer, in other words from 173 * one of the base DNs. 174 * @param entryContainer The entry container that holds the entries to be 175 * exported. 176 * @throws DatabaseException If an error occurs in the JE database. 177 * @throws IOException If an error occurs while writing an entry. 178 * @throws LDIFException If an error occurs while trying to determine 179 * whether to write an entry. 180 */ 181 private void exportContainer(EntryContainer entryContainer) 182 throws DatabaseException, IOException, LDIFException 183 { 184 ID2Entry id2entry = entryContainer.getID2Entry(); 185 186 Cursor cursor = id2entry.openCursor(null, new CursorConfig()); 187 try 188 { 189 DatabaseEntry key = new DatabaseEntry(); 190 DatabaseEntry data = new DatabaseEntry(); 191 192 OperationStatus status; 193 for (status = cursor.getFirst(key, data, LockMode.DEFAULT); 194 status == OperationStatus.SUCCESS; 195 status = cursor.getNext(key, data, LockMode.DEFAULT)) 196 { 197 if (exportConfig.isCancelled()) 198 { 199 break; 200 } 201 202 EntryID entryID = null; 203 try 204 { 205 entryID = new EntryID(key); 206 } 207 catch (Exception e) 208 { 209 if (logger.isTraceEnabled()) 210 { 211 logger.traceException(e); 212 213 logger.trace("Malformed id2entry ID %s.%n", 214 StaticUtils.bytesToHex(key.getData())); 215 } 216 skippedCount++; 217 continue; 218 } 219 220 if (entryID.longValue() == 0) 221 { 222 // This is the stored entry count. 223 continue; 224 } 225 226 Entry entry = null; 227 try 228 { 229 entry = ID2Entry.entryFromDatabase(ByteString.wrap(data.getData()), 230 entryContainer.getRootContainer().getCompressedSchema()); 231 } 232 catch (Exception e) 233 { 234 if (logger.isTraceEnabled()) 235 { 236 logger.traceException(e); 237 238 logger.trace("Malformed id2entry record for ID %d:%n%s%n", 239 entryID.longValue(), 240 StaticUtils.bytesToHex(data.getData())); 241 } 242 skippedCount++; 243 continue; 244 } 245 246 if (entry.toLDIF(exportConfig)) 247 { 248 exportedCount++; 249 } 250 else 251 { 252 skippedCount++; 253 } 254 } 255 } 256 finally 257 { 258 cursor.close(); 259 } 260 } 261 262 /** 263 * This class reports progress of the export job at fixed intervals. 264 */ 265 class ProgressTask extends TimerTask 266 { 267 /** 268 * The number of entries that had been exported at the time of the 269 * previous progress report. 270 */ 271 private long previousCount; 272 273 /** 274 * The time in milliseconds of the previous progress report. 275 */ 276 private long previousTime; 277 278 /** 279 * Create a new export progress task. 280 */ 281 public ProgressTask() 282 { 283 previousTime = System.currentTimeMillis(); 284 } 285 286 /** 287 * The action to be performed by this timer task. 288 */ 289 public void run() 290 { 291 long latestCount = exportedCount; 292 long deltaCount = latestCount - previousCount; 293 long latestTime = System.currentTimeMillis(); 294 long deltaTime = latestTime - previousTime; 295 296 if (deltaTime == 0) 297 { 298 return; 299 } 300 301 float rate = 1000f*deltaCount / deltaTime; 302 303 logger.info(NOTE_EXPORT_PROGRESS_REPORT, latestCount, skippedCount, rate); 304 305 previousCount = latestCount; 306 previousTime = latestTime; 307 } 308 } 309 310}