Importer.java revision 5707
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * CDDL HEADER START
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The contents of this file are subject to the terms of the
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Common Development and Distribution License, Version 1.0 only
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * (the "License"). You may not use this file except in compliance
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * with the License.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * You can obtain a copy of the license at
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * trunk/opends/resource/legal-notices/OpenDS.LICENSE
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * or https://OpenDS.dev.java.net/OpenDS.LICENSE.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * See the License for the specific language governing permissions
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * and limitations under the License.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * When distributing Covered Code, include this CDDL HEADER in each
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * file and include the License file at
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * trunk/opends/resource/legal-notices/OpenDS.LICENSE. If applicable,
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * add the following below this CDDL HEADER, with the fields enclosed
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * by brackets "[]" replaced with your own identifying information:
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Portions Copyright [yyyy] [name of copyright owner]
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * CDDL HEADER END
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Copyright 2008-2010 Sun Microsystems, Inc.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Portions Copyright 2011-2012 ForgeRock AS
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterpackage org.opends.server.backends.jeb.importLDIF;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.messages.JebMessages.*;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.loggers.ErrorLogger.logError;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.loggers.debug.DebugLogger.debugEnabled;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.loggers.debug.DebugLogger.getTracer;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.util.DynamicConstants.BUILD_ID;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.util.DynamicConstants.REVISION_NUMBER;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.util.ServerConstants.*;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.util.StaticUtils.getFileForPath;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport static org.opends.server.util.StaticUtils.stackTraceToString;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport java.util.concurrent.atomic.AtomicInteger;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.admin.std.meta.LocalDBIndexCfgDefn;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.admin.std.meta.LocalDBIndexCfgDefn.IndexType;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.admin.std.server.LocalDBBackendCfg;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.admin.std.server.LocalDBIndexCfg;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.api.DiskSpaceMonitorHandler;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.config.ConfigException;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.extensions.DiskSpaceMonitor;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterimport org.opends.server.loggers.debug.DebugTracer;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * This class provides the engine that performs both importing of LDIF files and
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * the rebuilding of indexes.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Fosterpublic final class Importer implements DiskSpaceMonitorHandler
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The tracer object for the debug logger.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final DebugTracer TRACER = getTracer();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int TIMER_INTERVAL = 10000;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final String DEFAULT_TMP_DIR = "import-tmp";
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final String TMPENV_DIR = "tmp-env";
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Defaults for DB cache.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int MAX_DB_CACHE_SIZE = 8 * MB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int MAX_DB_LOG_SIZE = 10 * MB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int MIN_DB_CACHE_SIZE = 4 * MB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Defaults for LDIF reader buffers, min memory required to import and default
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //size for byte buffers.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int READER_WRITER_BUFFER_SIZE = 8 * KB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int MIN_DB_CACHE_MEMORY = MAX_DB_CACHE_SIZE +
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int BYTE_BUFFER_CAPACITY = 128;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Min and MAX sizes of phase one buffer.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int MAX_BUFFER_SIZE = 2 * MB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int MIN_BUFFER_SIZE = 4 * KB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Min size of phase two read-ahead cache.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int MIN_READ_AHEAD_CACHE_SIZE = 2 * KB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Small heap threshold used to give more memory to JVM to attempt OOM errors.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private static final int SMALL_HEAP_SIZE = 256 * MB;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //The DN attribute type.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster static final IndexOutputBuffer.IndexComparator indexComparator =
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Phase one buffer and imported entries counts.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final AtomicInteger bufferCount = new AtomicInteger(0);
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final AtomicLong importCount = new AtomicLong(0);
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Phase one buffer size in bytes.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Temp scratch directory.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Index and thread counts.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final int indexCount;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Set to true when validation is skipped.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final boolean skipDNValidation;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Temporary environment used when DN validation is done in first phase.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Root container.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Import configuration.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final LDIFImportConfig importConfiguration;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Backend configuration.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final LocalDBBackendCfg backendConfiguration;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //LDIF reader.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Migrated entry count.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster // Size in bytes of temporary env.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster // Available memory at the start of the import.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster // Size in bytes of DB cache.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //The executor service used for the buffer sort tasks.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //The executor service used for the scratch file processing tasks.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private ExecutorService scratchFileWriterService;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Queue of free index buffers -- used to re-cycle index buffers;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final BlockingQueue<IndexOutputBuffer> freeBufferQueue =
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Map of index keys to index buffers. Used to allocate sorted
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //index buffers to a index writer thread.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster Map<IndexKey, BlockingQueue<IndexOutputBuffer>> indexKeyQueMap =
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster new ConcurrentHashMap<IndexKey, BlockingQueue<IndexOutputBuffer>>();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Map of DB containers to index managers. Used to start phase 2.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final List<IndexManager> indexMgrList =
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Map of DB containers to DN-based index managers. Used to start phase 2.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final List<IndexManager> DNIndexMgrList =
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Futures used to indicate when the index file writers are done flushing
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //their work queues and have exited. End of phase one.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final List<Future<?>> scratchFileWriterFutures;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //List of index file writer tasks. Used to signal stopScratchFileWriters to
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //the index file writer tasks when the LDIF file has been done.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final List<ScratchFileWriterTask> scratchFileWriterList;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Map of DNs to Suffix objects.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final Map<DN, Suffix> dnSuffixMap = new LinkedHashMap<DN, Suffix>();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Map of container ids to database containers.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final ConcurrentHashMap<Integer, DatabaseContainer> idContainerMap =
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster new ConcurrentHashMap<Integer, DatabaseContainer>();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Map of container ids to entry containers
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final ConcurrentHashMap<Integer, EntryContainer> idECMap =
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster new ConcurrentHashMap<Integer, EntryContainer>();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Used to synchronize when a scratch file index writer is first setup.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Rebuild index manager used when rebuilding indexes.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private final RebuildIndexManager rebuildManager;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Set to true if the backend was cleared.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private boolean clearedBackend = false;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Used to shutdown import if an error occurs in phase one.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private volatile boolean isCanceled = false;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private volatile boolean isPhaseOneDone = false;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster //Number of phase one buffers
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster if ((dnType = DirectoryServer.getAttributeType("dn")) == null)
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster dnType = DirectoryServer.getDefaultAttributeType("dn");
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Create a new import job with the specified rebuild index config.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param rebuildConfig
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The rebuild index configuration.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param cfg
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The local DB back-end configuration.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param envConfig
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The JEB environment config.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @throws InitializationException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * If a problem occurs during initialization.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @throws JebException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * If an error occurred when opening the DB.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @throws ConfigException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * If a problem occurs during initialization.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster public Importer(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg,
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster EnvironmentConfig envConfig) throws InitializationException,
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.rebuildManager = new RebuildIndexManager(rebuildConfig, cfg);
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.indexCount = rebuildManager.getIndexCount();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.scratchFileWriterList = new ArrayList<ScratchFileWriterTask>(
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.scratchFileWriterFutures = new CopyOnWriteArrayList<Future<?>>();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster parentDir = getFileForPath(rebuildConfig.getTmpDirectory());
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.tempDir = new File(parentDir, cfg.getBackendId());
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster Message message = ERR_JEB_IMPORT_CREATE_TMPDIR_ERROR.get(String
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Create a new import job with the specified ldif import config.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param importConfiguration
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The LDIF import configuration.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param localDBBackendCfg
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The local DB back-end configuration.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param envConfig
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The JEB environment config.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @throws InitializationException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * If a problem occurs during initialization.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @throws ConfigException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * If a problem occurs reading the configuration.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @throws DatabaseException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * If an error occurred when opening the DB.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster public Importer(LDIFImportConfig importConfiguration,
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster LocalDBBackendCfg localDBBackendCfg, EnvironmentConfig envConfig)
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster throws InitializationException, ConfigException, DatabaseException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.importConfiguration = importConfiguration;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.threadCount = Runtime.getRuntime().availableProcessors() * 2;
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.threadCount = importConfiguration.getThreadCount();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster // Determine the number of indexes.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster for (String indexName : localDBBackendCfg.listLocalDBIndexes())
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster LocalDBIndexCfg index = localDBBackendCfg.getLocalDBIndex(indexName);
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster SortedSet<IndexType> types = index.getIndexType();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster + index.getIndexExtensibleMatchingRule().size();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster if (!importConfiguration.appendToExistingData())
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.scratchFileWriterList = new ArrayList<ScratchFileWriterTask>(
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.scratchFileWriterFutures = new CopyOnWriteArrayList<Future<?>>();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster if (importConfiguration.getTmpDirectory() == null)
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster parentDir = getFileForPath(importConfiguration.getTmpDirectory());
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster this.tempDir = new File(parentDir, localDBBackendCfg.getBackendId());
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster Message message = ERR_JEB_IMPORT_CREATE_TMPDIR_ERROR.get(String
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster skipDNValidation = importConfiguration.getSkipDNValidation();
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster // Set up temporary environment.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Return the suffix instance in the specified map that matches the specified
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param dn The DN to search for.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param map The map to search.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @return The suffix instance that matches the DN, or null if no match is
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster public static Suffix getMatchSuffix(DN dn, Map<DN, Suffix> map)
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * Calculate buffer sizes and initialize JEB properties based on memory.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @param envConfig
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * The environment config to use in the calculations.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * @throws InitializationException
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster * If a problem occurs during calculation.
a688bcbb4bcff5398fdd29b86f83450257dc0df4Allan Foster private void initializeDBEnv(EnvironmentConfig envConfig)
throws InitializationException
else if (!clearedBackend)
if (!skipDNValidation)
if (!clearedBackend)
threadCount--;
private void calculateAvailableMemory()
final long totalAvailableMemory;
final long configuredMemory;
private void initializeIndexBuffers()
return null;
return null;
boolean keep = true;
keep = false;
if(!keep)
boolean keep = false;
keep = true;
if(!keep)
public LDIFImportResult
setIndexesTrusted(false);
phaseOne();
isPhaseOneDone = true;
if (!skipDNValidation)
if (isCanceled)
phaseTwo();
if (isCanceled)
setIndexesTrusted(true);
if(f.isDirectory())
recursiveDelete(f);
f.delete();
private void switchContainers()
private void processIndexFiles()
int readAheadSize;
int buffers;
dbThreads--;
readAheadSize)));
private void stopScratchFileWriters()
!isCanceled) {
suffix);
catch (Exception e)
message =
isCanceled =true;
return null;
null);
boolean found = false;
found = true;
if(!found) {
catch(Exception e)
message =
isCanceled =true;
return null;
return null;
catch(Exception e)
isCanceled = true;
return null;
if(!skipDNValidation)
return null;
catch (Exception e)
isCanceled = true;
return null;
if(!skipDNValidation)
if(!clearedBackend)
throws InterruptedException
return id;
return indexBuffer;
throws InterruptedException
byte[] dnBytes =
throws DatabaseException
private final int cacheSize;
private final int maxPermits;
private int remainingBuffers;
private volatile int totalBatches;
private int nextBufferID;
private int ownedPermits;
private volatile boolean isRunning = false;
isRunning = true;
return null;
return buffers;
public void endWriteTask()
isRunning = false;
if(!isCanceled)
if(!isCanceled)
.getBufferFileName());
if (isRunning)
/ bufferFileSize);
if(isCanceled)
return null;
if(isCanceled)
return null;
if (b.hasMoreData())
b.getNextRecord();
catch (Exception e)
e.printStackTrace();
endWriteTask();
return null;
int indexID)
class DNState
int parentIndex =
return null;
return parent;
return destBuffer;
status =
status =
return nodeID;
public void writeToDB()
boolean clearMap)
if(clearMap)
public void flush()
private boolean poisonSeen = false;
long bufferLen;
l.clear();
bufferCount++;
if(poisonSeen)
catch (IOException e)
isCanceled = true;
return null;
throws IOException
return bufferLen;
throws IOException
if(b.isPoison())
poisonSeen = true;
if(b.hasMoreData())
b.getNextRecord();
return bufferLen;
return packedSize;
|| isCanceled)
isCanceled = true;
return null;
.getIndexKey());
.getIndexKey());
return null;
throws FileNotFoundException
boolean isDN = false;
synchronized(synObj)
isDN = true;
if(isDN)
private long bufferFileSize;
private long totalDNS;
private final boolean isDN;
private final int limit;
return bufferFile;
private long getBufferFileSize()
return bufferFileSize;
return bufferIndexFile;
private long getDNCount()
return totalDNS;
private boolean isDN2ID()
return isDN;
return bufferFileName;
private int getLimit()
return limit;
private int getNumberOfBuffers()
return numberOfBuffers;
implements DiskSpaceMonitorHandler
private final boolean rebuildAll;
if(rebuildAll) {
if(isCanceled)
return null;
catch (Exception e)
isCanceled = true;
return null;
public void rebuildIndexes() throws
if (rebuildAll)
setAllIndexesTrusted(false);
setRebuildListIndexesTrusted(false);
phaseOne();
if (isCanceled)
phaseTwo();
if (rebuildAll)
setAllIndexesTrusted(true);
setRebuildListIndexesTrusted(true);
throws JebException
if(rebuildAll)
private void phaseTwo()
int indexCount;
if(!rebuildAll)
return indexCount;
return indexCount;
indexCount++;
indexCount++;
indexCount++;
indexCount++;
indexCount++;
indexCount++;
indexCount++;
boolean found = false;
found = true;
if(found)
if(!found) {
indexCount++;
indexCount++;
indexCount++;
indexCount++;
indexCount++;
indexCount++;
boolean shared = false;
indexCount++;
if(!shared)
shared=true;
indexCount++;
return indexCount;
clearDN2URI();
clearDN2URI();
throws DatabaseException
throws DatabaseException
public long getEntriesProcess()
public long getTotEntries()
return this.totalEntries;
isCanceled = true;
private long previousTime;
public void run()
long nCacheMiss =
catch (DatabaseException e)
private long previousTime;
private boolean evicting = false;
public FirstPhaseProgressTask()
catch (DatabaseException e)
throw new RuntimeException(e);
public void run()
if(skipDNValidation)
message =
long cleanerEntriesRead =
if (!evicting)
evicting = true;
message =
message =
message =
message =
catch (DatabaseException e)
private long previousTime;
private boolean evicting = false;
private long latestCount;
catch (DatabaseException e)
throw new RuntimeException(e);
public void run()
message =
if (!evicting)
evicting = true;
message =
message =
message =
catch (DatabaseException e)
public class EntryInformation
return suffix;
return entryID;
public enum ImportIndexType {
DN,
public class IndexKey {
private final int entryLimit;
int entryLimit)
public int hashCode()
return attributeType;
return indexType;
public int getEntryLimit()
return entryLimit;
public static interface DNCache {
private byte[] hashCode(byte[] b)
for (byte aB : b) {
throws JebException
throws JebException
boolean inserted = true;
inserted = false;
inserted = true;
return inserted;
boolean dnExists = false;
return dnExists;
throws DatabaseException
private class DefaultExceptionHandler
if (debugEnabled())
isCanceled = true;
if(!isPhaseOneDone)