Importer.java revision 4801
3339N/A/*
3339N/A * CDDL HEADER START
3339N/A *
3339N/A * The contents of this file are subject to the terms of the
3339N/A * Common Development and Distribution License, Version 1.0 only
3339N/A * (the "License"). You may not use this file except in compliance
3339N/A * with the License.
3339N/A *
3339N/A * You can obtain a copy of the license at
3339N/A * trunk/opends/resource/legal-notices/OpenDS.LICENSE
3339N/A * or https://OpenDS.dev.java.net/OpenDS.LICENSE.
3339N/A * See the License for the specific language governing permissions
3339N/A * and limitations under the License.
3339N/A *
3339N/A * When distributing Covered Code, include this CDDL HEADER in each
3339N/A * file and include the License file at
3339N/A * trunk/opends/resource/legal-notices/OpenDS.LICENSE. If applicable,
3339N/A * add the following below this CDDL HEADER, with the fields enclosed
3339N/A * by brackets "[]" replaced with your own identifying information:
3339N/A * Portions Copyright [yyyy] [name of copyright owner]
3339N/A *
3339N/A * CDDL HEADER END
3339N/A *
3339N/A *
4495N/A * Copyright 2008-2009 Sun Microsystems, Inc.
3339N/A */
3339N/A
3339N/Apackage org.opends.server.backends.jeb.importLDIF;
3339N/A
4591N/A
4591N/Aimport static org.opends.messages.JebMessages.*;
4591N/Aimport static org.opends.server.loggers.ErrorLogger.*;
4591N/Aimport static org.opends.server.util.DynamicConstants.*;
4591N/Aimport static org.opends.server.util.ServerConstants.*;
4591N/A
4591N/Aimport java.io.*;
4591N/Aimport java.nio.*;
4591N/Aimport java.nio.channels.FileChannel;
4591N/Aimport java.util.*;
4591N/Aimport java.util.concurrent.*;
4591N/Aimport java.util.concurrent.atomic.*;
4591N/A
4591N/Aimport static org.opends.server.util.StaticUtils.getFileForPath;
4591N/Aimport org.opends.messages.Message;
4591N/Aimport org.opends.messages.Category;
4591N/Aimport org.opends.messages.Severity;
3339N/Aimport org.opends.server.admin.std.server.LocalDBBackendCfg;
4765N/Aimport org.opends.server.admin.std.server.LocalDBIndexCfg;
4765N/Aimport org.opends.server.admin.std.meta.LocalDBIndexCfgDefn;
4591N/Aimport org.opends.server.backends.jeb.*;
3339N/Aimport org.opends.server.config.ConfigException;
3339N/Aimport org.opends.server.core.DirectoryServer;
4591N/Aimport org.opends.server.types.*;
4591N/Aimport org.opends.server.util.*;
4591N/Aimport com.sleepycat.je.*;
3339N/A
3339N/A
3339N/A/**
4765N/A * Performs LDIF import and rebuild of indexes.
3339N/A */
4591N/Apublic class Importer
4591N/A{
4591N/A private final int DRAIN_TO = 3;
4591N/A private final int TIMER_INTERVAL = 10000;
4736N/A private final int KB = 1024;
4736N/A private final int MB = (KB * KB);
4649N/A private final int LDIF_READER_BUFFER_SIZE = 2 * MB;
4649N/A private final int MIN_IMPORT_MEMORY_REQUIRED = 16 * MB;
4591N/A private final int MAX_BUFFER_SIZE = 48 * MB;
4591N/A private final int MIN_BUFFER_SIZE = 1024 * 100;
4591N/A private final int MIN_READ_AHEAD_CACHE_SIZE = 4096;
4591N/A private final int MAX_DB_CACHE_SIZE = 128 * MB;
4591N/A private final int MIN_DB_CACHE_SIZE = 16 * MB;
4649N/A private final int MAX_DB_LOG_BUFFER_BYTES = 100 * MB;
4643N/A private final int MEM_PCT_PHASE_1 = 45;
4736N/A private final int MEM_PCT_PHASE_2 = 55;
4736N/A private final int DN_STATE_CACHE_SIZE = 32 * KB;
3339N/A
4591N/A private final String DIRECT_PROPERTY = "import.directphase2";
4643N/A private static AttributeType dnType;
4643N/A private static IndexBuffer.DNComparator dnComparator
4643N/A = new IndexBuffer.DNComparator();
4643N/A private static final IndexBuffer.IndexComparator indexComparator =
4643N/A new IndexBuffer.IndexComparator();
3339N/A
4591N/A private final AtomicInteger bufferCount = new AtomicInteger(0);
4679N/A private final AtomicLong importCount = new AtomicLong(0);
4591N/A private final File tempDir;
4591N/A private final int indexCount, threadCount;
4643N/A private final boolean skipDNValidation;
4649N/A private final LDIFImportConfig importConfiguration;
4591N/A private final ByteBuffer directBuffer;
4591N/A private RootContainer rootContainer;
4591N/A private LDIFReader reader;
4643N/A private int bufferSize, indexBufferCount;
4643N/A private int migratedCount;
4649N/A private long dbCacheSize = 0, dbLogBufferSize = 0;
3339N/A
4591N/A //The executor service used for the sort tasks.
4591N/A private ExecutorService sortService;
3339N/A
4591N/A //The executor service used for the index processing tasks.
4591N/A private ExecutorService indexProcessService;
3339N/A
4591N/A //Queue of free index buffers -- used to re-cycle index buffers;
4649N/A private final BlockingQueue<IndexBuffer> freeBufferQueue =
4591N/A new LinkedBlockingQueue<IndexBuffer>();
3339N/A
4643N/A //Map of index keys to index buffers. Used to allocate sorted
4591N/A //index buffers to a index writer thread.
4591N/A private final
4643N/A Map<IndexKey, BlockingQueue<IndexBuffer>> indexKeyQueMap =
4643N/A new ConcurrentHashMap<IndexKey, BlockingQueue<IndexBuffer>>();
3339N/A
4591N/A //Map of DB containers to index managers. Used to start phase 2.
4643N/A private final List<IndexManager> indexMgrList =
4643N/A new LinkedList<IndexManager>();
3339N/A
4591N/A //Futures used to indicate when the index file writers are done flushing
4591N/A //their work queues and have exited. End of phase one.
4591N/A private final List<Future<?>> indexWriterFutures;
3339N/A
4591N/A //List of index file writer tasks. Used to signal stopIndexWriterTasks to the
4591N/A //index file writer tasks when the LDIF file has been done.
4591N/A private final List<IndexFileWriterTask> indexWriterList;
3598N/A
4643N/A
4643N/A //Map of DNs to Suffix objects.
4591N/A private final Map<DN, Suffix> dnSuffixMap = new LinkedHashMap<DN, Suffix>();
3681N/A
4643N/A
4643N/A private final ConcurrentHashMap<Integer, DatabaseContainer> idContainerMap =
4643N/A new ConcurrentHashMap<Integer, DatabaseContainer>();
4643N/A
4643N/A private final ConcurrentHashMap<Integer, EntryContainer> idECMap =
4643N/A new ConcurrentHashMap<Integer, EntryContainer>();
4643N/A
4643N/A private final Object synObj = new Object();
4765N/A private final RebuildManager rebuildManager;
4765N/A
4765N/A static
4643N/A {
4643N/A if ((dnType = DirectoryServer.getAttributeType("dn")) == null)
4643N/A {
4643N/A dnType = DirectoryServer.getDefaultAttributeType("dn");
4643N/A }
4643N/A }
4643N/A
4765N/A private void initialize()
4765N/A {
4765N/A
4765N/A }
4765N/A
4765N/A private
4765N/A Importer(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg,
4765N/A EnvironmentConfig envConfig) throws IOException,
4765N/A InitializationException, JebException, ConfigException
4765N/A {
4765N/A this.importConfiguration = null;
4765N/A this.threadCount = 1;
4765N/A this.rebuildManager = new RebuildManager(rebuildConfig, cfg);
4765N/A indexCount = rebuildManager.getIndexCount();
4765N/A indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
4765N/A indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
4765N/A File parentDir;
4765N/A if(rebuildConfig.getTmpDirectory() == null)
4765N/A {
4765N/A parentDir = getFileForPath("import-tmp");
4765N/A }
4765N/A else
4765N/A {
4765N/A parentDir = getFileForPath(rebuildConfig.getTmpDirectory());
4765N/A }
4765N/A tempDir = new File(parentDir, cfg.getBackendId());
4765N/A if(!tempDir.exists() && !tempDir.mkdirs())
4765N/A {
4765N/A Message message =
4765N/A ERR_JEB_IMPORT_CREATE_TMPDIR_ERROR.get(String.valueOf(tempDir));
4765N/A throw new IOException(message.toString());
4765N/A }
4765N/A if (tempDir.listFiles() != null)
4765N/A {
4765N/A for (File f : tempDir.listFiles())
4765N/A {
4765N/A f.delete();
4765N/A }
4765N/A }
4765N/A skipDNValidation = true;
4765N/A String propString = System.getProperty(DIRECT_PROPERTY);
4765N/A if(propString != null)
4765N/A {
4765N/A int directSize = Integer.valueOf(propString);
4765N/A directBuffer = ByteBuffer.allocateDirect(directSize);
4765N/A }
4765N/A else
4765N/A {
4765N/A directBuffer = null;
4765N/A }
4765N/A if(envConfig != null)
4765N/A {
4765N/A initializeDBEnv(envConfig);
4765N/A }
4765N/A }
4765N/A
4765N/A
3339N/A /**
3339N/A * Create a new import job with the specified ldif import config.
3339N/A *
4649N/A * @param importConfiguration The LDIF import configuration.
4765N/A * @param localDBBackendCfg The local DB back-end configuration.
4765N/A * @param envConfig The JEB environment config.
4591N/A * @throws IOException If a problem occurs while opening the LDIF file for
4591N/A * reading.
4649N/A * @throws InitializationException If a problem occurs during initialization.
3339N/A */
4765N/A private Importer(LDIFImportConfig importConfiguration,
4765N/A LocalDBBackendCfg localDBBackendCfg,
4765N/A EnvironmentConfig envConfig) throws IOException,
4765N/A InitializationException
4591N/A {
4765N/A this.rebuildManager = null;
4649N/A this.importConfiguration = importConfiguration;
4649N/A if(importConfiguration.getThreadCount() == 0)
4643N/A {
4643N/A threadCount = Runtime.getRuntime().availableProcessors() * 2;
4643N/A }
4643N/A else
4643N/A {
4649N/A threadCount = importConfiguration.getThreadCount();
4643N/A }
4765N/A indexCount = localDBBackendCfg.listLocalDBIndexes().length + 2;
4765N/A
4765N/A
4591N/A indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
4591N/A indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
4591N/A File parentDir;
4649N/A if(importConfiguration.getTmpDirectory() == null)
4591N/A {
4591N/A parentDir = getFileForPath("import-tmp");
3339N/A }
4591N/A else
4591N/A {
4649N/A parentDir = getFileForPath(importConfiguration.getTmpDirectory());
4591N/A }
4643N/A
4765N/A tempDir = new File(parentDir, localDBBackendCfg.getBackendId());
4591N/A if(!tempDir.exists() && !tempDir.mkdirs())
4591N/A {
4649N/A Message message =
4649N/A ERR_JEB_IMPORT_CREATE_TMPDIR_ERROR.get(String.valueOf(tempDir));
4649N/A throw new IOException(message.toString());
4591N/A }
4591N/A if (tempDir.listFiles() != null)
4591N/A {
4591N/A for (File f : tempDir.listFiles())
4591N/A {
4591N/A f.delete();
3339N/A }
3339N/A }
4649N/A skipDNValidation = importConfiguration.getSkipDNValidation();
4591N/A String propString = System.getProperty(DIRECT_PROPERTY);
4591N/A if(propString != null)
3339N/A {
4591N/A int directSize = Integer.valueOf(propString);
4591N/A directBuffer = ByteBuffer.allocateDirect(directSize);
3339N/A }
4591N/A else
4591N/A {
4591N/A directBuffer = null;
3339N/A }
4765N/A initializeDBEnv(envConfig);
3339N/A }
3339N/A
4765N/A /**
4765N/A * Return and import LDIF instance using the specified arguments.
4765N/A *
4765N/A * @param importCfg The import config to use.
4765N/A * @param localDBBackendCfg The local DB backend config to use.
4765N/A * @param envCfg The JEB environment config to use.
4765N/A * @return A import LDIF instance.
4765N/A *
4765N/A * @throws IOException If an I/O error occurs.
4765N/A * @throws InitializationException If the instance cannot be initialized.
4765N/A */
4765N/A public static
4765N/A Importer getInstance(LDIFImportConfig importCfg,
4765N/A LocalDBBackendCfg localDBBackendCfg,
4765N/A EnvironmentConfig envCfg)
4765N/A throws IOException, InitializationException
4765N/A {
4765N/A return new Importer(importCfg, localDBBackendCfg, envCfg);
4765N/A }
4765N/A
4765N/A
4765N/A /**
4765N/A * Return an import rebuild index instance using the specified arguments.
4765N/A *
4765N/A * @param rebuildCfg The rebuild config to use.
4765N/A * @param localDBBackendCfg The local DB backend config to use.
4765N/A * @param envCfg The JEB environment config to use.
4765N/A * @return An import rebuild index instance.
4765N/A *
4765N/A * @throws IOException If an I/O error occurs.
4765N/A * @throws InitializationException If the instance cannot be initialized.
4765N/A * @throws JebException If a JEB exception occurs.
4765N/A * @throws ConfigException If the instance cannot be configured.
4765N/A */
4765N/A public static synchronized
4765N/A Importer getInstance(RebuildConfig rebuildCfg,
4765N/A LocalDBBackendCfg localDBBackendCfg,
4765N/A EnvironmentConfig envCfg)
4765N/A throws IOException, InitializationException, JebException, ConfigException
4765N/A {
4765N/A return new Importer(rebuildCfg, localDBBackendCfg, envCfg);
4765N/A }
4765N/A
4765N/A private void getBufferSizes(long availMem, int buffers)
4591N/A {
4649N/A long memory = availMem - (MAX_DB_CACHE_SIZE + MAX_DB_LOG_BUFFER_BYTES);
4649N/A bufferSize = (int) (memory/buffers);
4591N/A if(bufferSize >= MIN_BUFFER_SIZE)
3615N/A {
4591N/A dbCacheSize = MAX_DB_CACHE_SIZE;
4649N/A dbLogBufferSize = MAX_DB_LOG_BUFFER_BYTES;
4591N/A if(bufferSize > MAX_BUFFER_SIZE)
3339N/A {
4591N/A bufferSize = MAX_BUFFER_SIZE;
3339N/A }
3339N/A }
4591N/A else
3339N/A {
4649N/A memory = availMem - MIN_DB_CACHE_SIZE - (MIN_DB_CACHE_SIZE * 7) / 100;
4649N/A bufferSize = (int) (memory/buffers);
4591N/A dbCacheSize = MIN_DB_CACHE_SIZE;
4591N/A if(bufferSize < MIN_BUFFER_SIZE)
3339N/A {
4649N/A Message message =
4610N/A NOTE_JEB_IMPORT_LDIF_BUFF_SIZE_LESS_DEFAULT.get(MIN_BUFFER_SIZE);
4649N/A logError(message);
4591N/A bufferSize = MIN_BUFFER_SIZE;
3339N/A }
4591N/A else
4591N/A {
4649N/A long constrainedMemory = memory - (buffers * MIN_BUFFER_SIZE);
4591N/A bufferSize = (int) ((buffers * MIN_BUFFER_SIZE) +
4649N/A (constrainedMemory * 50/100));
4591N/A bufferSize /= buffers;
4649N/A dbCacheSize = MIN_DB_CACHE_SIZE + (constrainedMemory * 50/100);
3339N/A }
3339N/A }
3339N/A }
3339N/A
3339N/A /**
4591N/A * Return the suffix instance in the specified map that matches the specified
4591N/A * DN.
4591N/A *
4591N/A * @param dn The DN to search for.
4591N/A * @param map The map to search.
4591N/A * @return The suffix instance that matches the DN, or null if no match is
4591N/A * found.
4591N/A */
4591N/A public static Suffix getMatchSuffix(DN dn, Map<DN, Suffix> map)
4591N/A {
4591N/A Suffix suffix = null;
4591N/A DN nodeDN = dn;
4591N/A
4591N/A while (suffix == null && nodeDN != null) {
4591N/A suffix = map.get(nodeDN);
4591N/A if (suffix == null)
4591N/A {
4591N/A nodeDN = nodeDN.getParentDNInSuffix();
4591N/A }
4591N/A }
4591N/A return suffix;
4591N/A }
4591N/A
4591N/A /**
4591N/A * Calculate buffer sizes and initialize JEB properties based on memory.
4591N/A *
4591N/A * @param envConfig The environment config to use in the calculations.
3339N/A *
4591N/A * @throws InitializationException If a problem occurs during calculation.
4591N/A */
4765N/A private void initializeDBEnv(EnvironmentConfig envConfig)
4591N/A throws InitializationException
4591N/A {
4649N/A Message message;
4649N/A Runtime runTime = Runtime.getRuntime();
4649N/A long freeMemory = runTime.freeMemory();
4649N/A long maxMemory = runTime.maxMemory();
4649N/A long totMemory = runTime.totalMemory();
4649N/A long totFreeMemory = (freeMemory + (maxMemory - totMemory));
4649N/A long availableMemoryImport = (totFreeMemory * MEM_PCT_PHASE_1) / 100;
4649N/A int phaseOneBuffers = 2 * (indexCount * threadCount);
4649N/A message = NOTE_JEB_IMPORT_LDIF_TOT_MEM_BUF.get(availableMemoryImport,
4649N/A phaseOneBuffers);
4649N/A logError(message);
4649N/A if (System.getProperty(PROPERTY_RUNNING_UNIT_TESTS) == null)
4649N/A {
4649N/A if (availableMemoryImport < MIN_IMPORT_MEMORY_REQUIRED)
4649N/A {
4649N/A message = ERR_IMPORT_LDIF_LACK_MEM.get(16);
4649N/A throw new InitializationException(message);
4649N/A }
4649N/A }
4649N/A getBufferSizes(availableMemoryImport, phaseOneBuffers);
4649N/A envConfig.setConfigParam("je.maxMemory", Long.toString(dbCacheSize));
4649N/A message =
4649N/A NOTE_JEB_IMPORT_LDIF_DB_MEM_BUF_INFO.get(dbCacheSize, bufferSize);
4649N/A logError(message);
4649N/A if(dbLogBufferSize != 0)
4649N/A {
4649N/A envConfig.setConfigParam("je.log.totalBufferBytes",
4649N/A Long.toString(dbLogBufferSize));
4649N/A message = NOTE_JEB_IMPORT_LDIF_LOG_BYTES.get(dbLogBufferSize);
4649N/A logError(message);
4649N/A }
4591N/A }
4591N/A
4591N/A
4649N/A private void initializeIndexBuffers(int threadCount)
4591N/A {
4643N/A indexBufferCount = 2 * (indexCount * threadCount);
4643N/A for(int i = 0; i < indexBufferCount; i++)
4591N/A {
4591N/A IndexBuffer b = IndexBuffer.createIndexBuffer(bufferSize);
4649N/A freeBufferQueue.add(b);
4591N/A }
4591N/A }
4591N/A
4591N/A
4591N/A
4649N/A private void initializeSuffixes() throws DatabaseException, JebException,
4643N/A ConfigException, InitializationException
4591N/A {
4643N/A for(EntryContainer ec : rootContainer.getEntryContainers())
4643N/A {
4643N/A Suffix suffix = getSuffix(ec);
4643N/A if(suffix != null)
4643N/A {
4643N/A dnSuffixMap.put(ec.getBaseDN(), suffix);
4643N/A }
4643N/A }
4591N/A }
4591N/A
4591N/A
4643N/A private Suffix getSuffix(EntryContainer entryContainer)
4643N/A throws DatabaseException, JebException, ConfigException,
4643N/A InitializationException {
4643N/A DN baseDN = entryContainer.getBaseDN();
4649N/A EntryContainer sourceEntryContainer = null;
4643N/A List<DN> includeBranches = new ArrayList<DN>();
4643N/A List<DN> excludeBranches = new ArrayList<DN>();
4643N/A
4649N/A if(!importConfiguration.appendToExistingData() &&
4649N/A !importConfiguration.clearBackend())
4643N/A {
4649N/A for(DN dn : importConfiguration.getExcludeBranches())
4643N/A {
4643N/A if(baseDN.equals(dn))
4643N/A {
4643N/A // This entire base DN was explicitly excluded. Skip.
4643N/A return null;
4643N/A }
4643N/A if(baseDN.isAncestorOf(dn))
4643N/A {
4643N/A excludeBranches.add(dn);
4643N/A }
4643N/A }
4643N/A
4649N/A if(!importConfiguration.getIncludeBranches().isEmpty())
4643N/A {
4649N/A for(DN dn : importConfiguration.getIncludeBranches())
4643N/A {
4643N/A if(baseDN.isAncestorOf(dn))
4643N/A {
4643N/A includeBranches.add(dn);
4643N/A }
4643N/A }
4643N/A
4643N/A if(includeBranches.isEmpty())
4643N/A {
4649N/A /*
4649N/A There are no branches in the explicitly defined include list under
4649N/A this base DN. Skip this base DN all together.
4649N/A */
4643N/A
4649N/A return null;
4643N/A }
4643N/A
4643N/A // Remove any overlapping include branches.
4643N/A Iterator<DN> includeBranchIterator = includeBranches.iterator();
4643N/A while(includeBranchIterator.hasNext())
4643N/A {
4643N/A DN includeDN = includeBranchIterator.next();
4643N/A boolean keep = true;
4643N/A for(DN dn : includeBranches)
4643N/A {
4643N/A if(!dn.equals(includeDN) && dn.isAncestorOf(includeDN))
4643N/A {
4643N/A keep = false;
4643N/A break;
4643N/A }
4643N/A }
4643N/A if(!keep)
4643N/A {
4643N/A includeBranchIterator.remove();
4643N/A }
4643N/A }
4643N/A
4649N/A // Remove any exclude branches that are not are not under a include
4643N/A // branch since they will be migrated as part of the existing entries
4643N/A // outside of the include branches anyways.
4643N/A Iterator<DN> excludeBranchIterator = excludeBranches.iterator();
4643N/A while(excludeBranchIterator.hasNext())
4643N/A {
4643N/A DN excludeDN = excludeBranchIterator.next();
4643N/A boolean keep = false;
4643N/A for(DN includeDN : includeBranches)
4643N/A {
4643N/A if(includeDN.isAncestorOf(excludeDN))
4643N/A {
4643N/A keep = true;
4643N/A break;
4643N/A }
4643N/A }
4643N/A if(!keep)
4643N/A {
4643N/A excludeBranchIterator.remove();
4643N/A }
4643N/A }
4643N/A
4643N/A if(includeBranches.size() == 1 && excludeBranches.size() == 0 &&
4643N/A includeBranches.get(0).equals(baseDN))
4643N/A {
4643N/A // This entire base DN is explicitly included in the import with
4643N/A // no exclude branches that we need to migrate. Just clear the entry
4643N/A // container.
4643N/A entryContainer.lock();
4643N/A entryContainer.clear();
4643N/A entryContainer.unlock();
4643N/A }
4643N/A else
4643N/A {
4643N/A // Create a temp entry container
4649N/A sourceEntryContainer = entryContainer;
4643N/A entryContainer =
4643N/A rootContainer.openEntryContainer(baseDN,
4643N/A baseDN.toNormalizedString() +
4643N/A "_importTmp");
4643N/A }
4643N/A }
4643N/A }
4649N/A return Suffix.createSuffixContext(entryContainer, sourceEntryContainer,
4643N/A includeBranches, excludeBranches);
4643N/A }
4643N/A
4643N/A
4765N/A /**
4765N/A * Rebuild the indexes using the specified rootcontainer.
4765N/A *
4765N/A * @param rootContainer The rootcontainer to rebuild indexes in.
4765N/A *
4765N/A * @throws ConfigException If a configuration error occurred.
4765N/A * @throws InitializationException If an initialization error occurred.
4765N/A * @throws IOException If an IO error occurred.
4765N/A * @throws JebException If the JEB database had an error.
4765N/A * @throws DatabaseException If a database error occurred.
4765N/A * @throws InterruptedException If an interrupted error occurred.
4765N/A * @throws ExecutionException If an execution error occurred.
4765N/A */
4765N/A public void
4765N/A rebuildIndexes(RootContainer rootContainer) throws ConfigException,
4765N/A InitializationException, IOException, JebException, DatabaseException,
4765N/A InterruptedException, ExecutionException
4765N/A {
4765N/A this.rootContainer = rootContainer;
4765N/A long startTime = System.currentTimeMillis();
4765N/A rebuildManager.initialize();
4765N/A rebuildManager.printStartMessage();
4765N/A rebuildManager.rebuldIndexes();
4765N/A tempDir.delete();
4765N/A rebuildManager.printStopMessage(startTime);
4765N/A }
4765N/A
4591N/A
4591N/A /**
4649N/A * Import a LDIF using the specified root container.
4591N/A *
4591N/A * @param rootContainer The root container to use during the import.
4591N/A *
4591N/A * @return A LDIF result.
4591N/A * @throws ConfigException If the import failed because of an configuration
4591N/A * error.
4591N/A * @throws IOException If the import failed because of an IO error.
4591N/A * @throws InitializationException If the import failed because of an
4591N/A * initialization error.
4591N/A * @throws JebException If the import failed due to a database error.
4591N/A * @throws InterruptedException If the import failed due to an interrupted
4591N/A * error.
4591N/A * @throws ExecutionException If the import failed due to an execution error.
4643N/A * @throws DatabaseException If the import failed due to a database error.
3339N/A */
4591N/A public LDIFImportResult
4591N/A processImport(RootContainer rootContainer) throws ConfigException,
4643N/A InitializationException, IOException, JebException, DatabaseException,
4591N/A InterruptedException, ExecutionException
4591N/A {
4801N/A this.rootContainer = rootContainer;
4801N/A this.reader = new LDIFReader(importConfiguration, rootContainer,
4801N/A LDIF_READER_BUFFER_SIZE);
4660N/A try
4660N/A {
4660N/A Message message =
4660N/A NOTE_JEB_IMPORT_STARTING.get(DirectoryServer.getVersionString(),
4660N/A BUILD_ID, REVISION_NUMBER);
4660N/A logError(message);
4660N/A message = NOTE_JEB_IMPORT_THREAD_COUNT.get(threadCount);
4660N/A logError(message);
4660N/A initializeSuffixes();
4660N/A long startTime = System.currentTimeMillis();
4660N/A processPhaseOne();
4660N/A processPhaseTwo();
4660N/A setIndexesTrusted();
4660N/A switchContainers();
4660N/A tempDir.delete();
4660N/A long finishTime = System.currentTimeMillis();
4660N/A long importTime = (finishTime - startTime);
4660N/A float rate = 0;
4660N/A if (importTime > 0)
4660N/A rate = 1000f * reader.getEntriesRead() / importTime;
4765N/A message = NOTE_JEB_IMPORT_FINAL_STATUS.get(reader.getEntriesRead(),
4765N/A importCount.get(), reader.getEntriesIgnored(),
4765N/A reader.getEntriesRejected(), migratedCount,
4765N/A importTime / 1000, rate);
4660N/A logError(message);
4649N/A }
4643N/A finally
4643N/A {
4643N/A reader.close();
4643N/A }
4591N/A return new LDIFImportResult(reader.getEntriesRead(), reader
4591N/A .getEntriesRejected(), reader.getEntriesIgnored());
4591N/A }
4591N/A
4591N/A
4643N/A private void switchContainers() throws DatabaseException, JebException {
4643N/A
4643N/A for(Suffix suffix : dnSuffixMap.values()) {
4643N/A DN baseDN = suffix.getBaseDN();
4649N/A EntryContainer entryContainer =
4643N/A suffix.getSrcEntryContainer();
4649N/A if(entryContainer != null) {
4649N/A EntryContainer needRegisterContainer =
4643N/A rootContainer.unregisterEntryContainer(baseDN);
4643N/A //Make sure the unregistered EC for the base DN is the same as
4643N/A //the one in the import context.
4649N/A if(needRegisterContainer != needRegisterContainer) {
4649N/A rootContainer.registerEntryContainer(baseDN, needRegisterContainer);
4643N/A continue;
4643N/A }
4649N/A needRegisterContainer.lock();
4649N/A needRegisterContainer.close();
4649N/A needRegisterContainer.delete();
4649N/A needRegisterContainer.unlock();
4643N/A EntryContainer newEC = suffix.getEntryContainer();
4643N/A newEC.lock();
4643N/A newEC.setDatabasePrefix(baseDN.toNormalizedString());
4643N/A newEC.unlock();
4643N/A rootContainer.registerEntryContainer(baseDN, newEC);
4643N/A }
4643N/A }
4643N/A }
4643N/A
4643N/A
4643N/A
4591N/A private void setIndexesTrusted() throws JebException
4591N/A {
4591N/A try {
4591N/A for(Suffix s : dnSuffixMap.values()) {
4591N/A s.setIndexesTrusted();
4591N/A }
4591N/A }
4591N/A catch (DatabaseException ex)
4591N/A {
4649N/A Message message =
4649N/A NOTE_JEB_IMPORT_LDIF_TRUSTED_FAILED.get(ex.getMessage());
4649N/A throw new JebException(message);
4591N/A }
4591N/A }
4591N/A
4591N/A
4591N/A private void processPhaseOne() throws InterruptedException, ExecutionException
4591N/A {
4649N/A initializeIndexBuffers(threadCount);
4591N/A FirstPhaseProgressTask progressTask = new FirstPhaseProgressTask();
4591N/A Timer timer = new Timer();
4591N/A timer.scheduleAtFixedRate(progressTask, TIMER_INTERVAL, TIMER_INTERVAL);
4591N/A indexProcessService = Executors.newFixedThreadPool(2 * indexCount);
4591N/A sortService = Executors.newFixedThreadPool(threadCount);
4643N/A ExecutorService execService = Executors.newFixedThreadPool(threadCount);
4643N/A List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(threadCount);
4591N/A
4643N/A tasks.add(new MigrateExistingTask());
4643N/A List<Future<Void>> results = execService.invokeAll(tasks);
4765N/A for (Future<Void> result : results) {
4765N/A if(!result.isDone()) {
4765N/A result.get();
4765N/A }
4765N/A }
4643N/A tasks.clear();
4643N/A results.clear();
4643N/A
4649N/A if (importConfiguration.appendToExistingData() &&
4649N/A importConfiguration.replaceExistingEntries())
4591N/A {
4643N/A for (int i = 0; i < threadCount; i++)
4643N/A {
4643N/A tasks.add(new AppendReplaceTask());
4643N/A }
4643N/A }
4643N/A else
4643N/A {
4643N/A for (int i = 0; i < threadCount; i++)
4643N/A {
4643N/A tasks.add(new ImportTask());
4643N/A }
4591N/A }
4643N/A results = execService.invokeAll(tasks);
4643N/A for (Future<Void> result : results)
4765N/A if(!result.isDone()) {
4765N/A result.get();
4765N/A }
4643N/A tasks.clear();
4643N/A results.clear();
4643N/A tasks.add(new MigrateExcludedTask());
4643N/A results = execService.invokeAll(tasks);
4643N/A for (Future<Void> result : results)
4765N/A if(!result.isDone()) {
4765N/A result.get();
4765N/A }
4591N/A stopIndexWriterTasks();
4591N/A for (Future<?> result : indexWriterFutures)
4591N/A {
4765N/A if(!result.isDone()) {
4765N/A result.get();
4765N/A }
4591N/A }
4717N/A indexWriterList.clear();
4718N/A indexWriterFutures.clear();
4718N/A indexKeyQueMap.clear();
4591N/A execService.shutdown();
4649N/A freeBufferQueue.clear();
4591N/A sortService.shutdown();
4591N/A timer.cancel();
4591N/A }
4591N/A
4591N/A
3339N/A
4591N/A private void processPhaseTwo() throws InterruptedException
4591N/A {
4591N/A SecondPhaseProgressTask progress2Task =
4765N/A new SecondPhaseProgressTask(indexMgrList, reader.getEntriesRead());
4591N/A Timer timer2 = new Timer();
4591N/A timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
4591N/A processIndexFiles();
4591N/A timer2.cancel();
4591N/A }
4591N/A
4591N/A
4591N/A
4591N/A private void processIndexFiles() throws InterruptedException
4591N/A {
4591N/A List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(indexCount);
4591N/A if(bufferCount.get() == 0)
4591N/A {
4591N/A return;
4591N/A }
4591N/A int cacheSize = cacheSizeFromFreeMemory();
4591N/A int p = 0;
4591N/A int offSet = 0;
4591N/A if(directBuffer != null)
4591N/A {
4591N/A cacheSize = cacheSizeFromDirectMemory();
4591N/A }
4643N/A for(IndexManager idxMgr : indexMgrList)
4591N/A {
4591N/A if(directBuffer != null)
4591N/A {
4643N/A int cacheSizes = cacheSize * idxMgr.getBufferList().size();
4591N/A offSet += cacheSizes;
4591N/A directBuffer.limit(offSet);
4591N/A directBuffer.position(p);
4591N/A ByteBuffer b = directBuffer.slice();
4643N/A tasks.add(new IndexWriteDBTask(idxMgr, b, cacheSize));
4591N/A p += cacheSizes;
3339N/A }
4591N/A else
4591N/A {
4643N/A tasks.add(new IndexWriteDBTask(idxMgr, null, cacheSize));
4591N/A }
4591N/A }
4591N/A List<Future<Void>> results = indexProcessService.invokeAll(tasks);
4591N/A for (Future<Void> result : results)
4591N/A assert result.isDone();
4591N/A indexProcessService.shutdown();
4591N/A }
4591N/A
4591N/A
4591N/A private int cacheSizeFromDirectMemory()
4591N/A {
4649N/A int cacheSize = directBuffer.capacity()/bufferCount.get();
4591N/A if(cacheSize > bufferSize)
4591N/A {
4591N/A cacheSize = bufferSize;
4591N/A }
4649N/A Message message =
4594N/A NOTE_JEB_IMPORT_LDIF_DIRECT_MEM_REPORT.get(bufferCount.get(), cacheSize);
4649N/A logError(message);
4591N/A return cacheSize;
4591N/A }
4591N/A
4591N/A private int cacheSizeFromFreeMemory()
4591N/A {
4649N/A Runtime runTime = Runtime.getRuntime();
4649N/A long freeMemory = runTime.freeMemory();
4649N/A long maxMemory = runTime.maxMemory();
4649N/A long totMemory = runTime.totalMemory();
4610N/A long totFreeMemory = (freeMemory + (maxMemory - totMemory));
4610N/A long availMemory = (totFreeMemory * MEM_PCT_PHASE_2) / 100;
4649N/A int averageBufferSize = (int)(availMemory / bufferCount.get());
4649N/A int cacheSize = Math.max(MIN_READ_AHEAD_CACHE_SIZE, averageBufferSize);
4591N/A if(cacheSize > bufferSize)
4591N/A {
4591N/A cacheSize = bufferSize;
4591N/A }
4649N/A Message message =
4594N/A NOTE_JEB_IMPORT_LDIF_INDIRECT_MEM_REPORT.get(bufferCount.get(), cacheSize);
4649N/A logError(message);
4591N/A return cacheSize;
4591N/A }
4591N/A
4591N/A
4591N/A private void stopIndexWriterTasks()
4591N/A {
4649N/A IndexBuffer indexBuffer = IndexBuffer.createIndexBuffer(0);
4591N/A for(IndexFileWriterTask task : indexWriterList)
4591N/A {
4649N/A task.queue.add(indexBuffer);
3339N/A }
3339N/A }
3339N/A
3339N/A
4660N/A /**
4643N/A * Task used to migrate excluded branch.
4591N/A */
4643N/A private final class MigrateExcludedTask extends ImportTask
4643N/A {
4643N/A
4643N/A public Void call() throws Exception
4643N/A {
4643N/A for(Suffix suffix : dnSuffixMap.values()) {
4649N/A EntryContainer entryContainer = suffix.getSrcEntryContainer();
4649N/A if(entryContainer != null &&
4643N/A !suffix.getExcludeBranches().isEmpty()) {
4643N/A DatabaseEntry key = new DatabaseEntry();
4643N/A DatabaseEntry data = new DatabaseEntry();
4643N/A LockMode lockMode = LockMode.DEFAULT;
4643N/A OperationStatus status;
4643N/A Message message = NOTE_JEB_IMPORT_MIGRATION_START.get(
4643N/A "excluded", String.valueOf(suffix.getBaseDN()));
4643N/A logError(message);
4643N/A Cursor cursor =
4649N/A entryContainer.getDN2ID().openCursor(null,
4643N/A CursorConfig.READ_COMMITTED);
4649N/A Comparator<byte[]> comparator =
4649N/A entryContainer.getDN2ID().getComparator();
4643N/A try {
4643N/A for(DN excludedDN : suffix.getExcludeBranches()) {
4643N/A byte[] bytes =
4643N/A StaticUtils.getBytes(excludedDN.toNormalizedString());
4643N/A key.setData(bytes);
4643N/A status = cursor.getSearchKeyRange(key, data, lockMode);
4643N/A if(status == OperationStatus.SUCCESS &&
4643N/A Arrays.equals(key.getData(), bytes)) {
4643N/A // This is the base entry for a branch that was excluded in the
4643N/A // import so we must migrate all entries in this branch over to
4643N/A // the new entry container.
4660N/A byte[] end = StaticUtils.getBytes("," +
4660N/A excludedDN.toNormalizedString());
4643N/A end[0] = (byte) (end[0] + 1);
4643N/A
4643N/A while(status == OperationStatus.SUCCESS &&
4649N/A comparator.compare(key.getData(), end) < 0 &&
4649N/A !importConfiguration.isCancelled()) {
4643N/A EntryID id = new EntryID(data);
4649N/A Entry entry = entryContainer.getID2Entry().get(null,
4643N/A id, LockMode.DEFAULT);
4643N/A processEntry(entry, rootContainer.getNextEntryID(),
4643N/A suffix);
4643N/A migratedCount++;
4643N/A status = cursor.getNext(key, data, lockMode);
4643N/A }
4643N/A }
4643N/A }
4643N/A }
4660N/A catch (Exception e)
4660N/A {
4660N/A message =
4660N/A ERR_JEB_IMPORT_LDIF_MIGRATE_EXCLUDED_TASK_ERR.get(e.getMessage());
4660N/A logError(message);
4660N/A throw e;
4660N/A }
4643N/A finally
4643N/A {
4643N/A cursor.close();
4643N/A flushIndexBuffers();
4643N/A closeCursors();
4643N/A }
4643N/A }
4643N/A }
4643N/A return null;
4643N/A }
4643N/A }
4643N/A
4643N/A
4643N/A /**
4643N/A * Task to migrate existing entries.
4643N/A */
4643N/A private final class MigrateExistingTask extends ImportTask
4643N/A {
4643N/A
4643N/A public Void call() throws Exception
4643N/A {
4660N/A for(Suffix suffix : dnSuffixMap.values()) {
4660N/A EntryContainer entryContainer = suffix.getSrcEntryContainer();
4660N/A if(entryContainer != null &&
4660N/A !suffix.getIncludeBranches().isEmpty()) {
4660N/A DatabaseEntry key = new DatabaseEntry();
4660N/A DatabaseEntry data = new DatabaseEntry();
4660N/A LockMode lockMode = LockMode.DEFAULT;
4660N/A OperationStatus status;
4660N/A Message message = NOTE_JEB_IMPORT_MIGRATION_START.get(
4660N/A "existing", String.valueOf(suffix.getBaseDN()));
4660N/A logError(message);
4660N/A Cursor cursor =
4660N/A entryContainer.getDN2ID().openCursor(null,
4660N/A null);
4660N/A try {
4660N/A status = cursor.getFirst(key, data, lockMode);
4660N/A while(status == OperationStatus.SUCCESS &&
4649N/A !importConfiguration.isCancelled()) {
4660N/A DN dn = DN.decode(ByteString.wrap(key.getData()));
4660N/A if(!suffix.getIncludeBranches().contains(dn)) {
4660N/A EntryID id = new EntryID(data);
4660N/A Entry entry =
4660N/A entryContainer.getID2Entry().get(null,
4660N/A id, LockMode.DEFAULT);
4660N/A processEntry(entry, rootContainer.getNextEntryID(),suffix);
4660N/A migratedCount++;
4660N/A status = cursor.getNext(key, data, lockMode);
4660N/A } else {
4660N/A // This is the base entry for a branch that will be included
4660N/A // in the import so we don't want to copy the branch to the
4660N/A // new entry container.
4643N/A
4660N/A /**
4660N/A * Advance the cursor to next entry at the same level in the
4660N/A * DIT
4660N/A * skipping all the entries in this branch.
4660N/A * Set the next starting value to a value of equal length but
4660N/A * slightly greater than the previous DN. Since keys are
4660N/A * compared in reverse order we must set the first byte
4660N/A * (the comma).
4660N/A * No possibility of overflow here.
4660N/A */
4660N/A byte[] begin =
4660N/A StaticUtils.getBytes("," + dn.toNormalizedString());
4660N/A begin[0] = (byte) (begin[0] + 1);
4660N/A key.setData(begin);
4660N/A status = cursor.getSearchKeyRange(key, data, lockMode);
4643N/A }
4643N/A }
4643N/A }
4660N/A catch(Exception e)
4660N/A {
4660N/A message =
4660N/A ERR_JEB_IMPORT_LDIF_MIGRATE_EXISTING_TASK_ERR.get(e.getMessage());
4660N/A logError(message);
4660N/A throw e;
4660N/A }
4660N/A finally
4660N/A {
4660N/A cursor.close();
4660N/A flushIndexBuffers();
4660N/A closeCursors();
4660N/A }
4643N/A }
4660N/A }
4643N/A return null;
4643N/A }
4643N/A }
4643N/A
4643N/A /**
4643N/A * Task to handle append/replace combination.
4643N/A */
4643N/A private class AppendReplaceTask extends ImportTask
4643N/A {
4643N/A private final Set<byte[]> insertKeySet = new HashSet<byte[]>();
4643N/A private final Set<byte[]> deleteKeySet = new HashSet<byte[]>();
4643N/A private final EntryInformation entryInfo = new EntryInformation();
4643N/A private Entry oldEntry;
4643N/A private EntryID entryID;
4591N/A
4591N/A /**
4591N/A * {@inheritDoc}
4591N/A */
4591N/A public Void call() throws Exception
4591N/A {
4660N/A try
4591N/A {
4660N/A while (true)
4591N/A {
4660N/A if (importConfiguration.isCancelled())
4660N/A {
4660N/A IndexBuffer indexBuffer = IndexBuffer.createIndexBuffer(0);
4660N/A freeBufferQueue.add(indexBuffer);
4660N/A return null;
4660N/A }
4660N/A oldEntry = null;
4660N/A Entry entry = reader.readEntry(dnSuffixMap, entryInfo);
4660N/A if (entry == null)
4660N/A {
4660N/A break;
4660N/A }
4660N/A entryID = entryInfo.getEntryID();
4660N/A Suffix suffix = entryInfo.getSuffix();
4660N/A processEntry(entry, suffix);
4591N/A }
4660N/A flushIndexBuffers();
4660N/A closeCursors();
4643N/A }
4660N/A catch(Exception e)
4660N/A {
4660N/A Message message =
4660N/A ERR_JEB_IMPORT_LDIF_APPEND_REPLACE_TASK_ERR.get(e.getMessage());
4660N/A logError(message);
4660N/A throw e;
4660N/A }
4643N/A return null;
4643N/A }
4643N/A
4643N/A
4643N/A void processEntry(Entry entry, Suffix suffix)
4643N/A throws DatabaseException, ConfigException, DirectoryException,
4643N/A JebException
4643N/A
4643N/A {
4643N/A DN entryDN = entry.getDN();
4643N/A DN2ID dn2id = suffix.getDN2ID();
4643N/A EntryID oldID = dn2id.get(null, entryDN, LockMode.DEFAULT);
4643N/A if(oldID != null)
4643N/A {
4643N/A oldEntry = suffix.getID2Entry().get(null, oldID, LockMode.DEFAULT);
4643N/A }
4643N/A if(oldEntry == null)
4643N/A {
4643N/A if(!skipDNValidation)
4591N/A {
4591N/A if(!processParent(entryDN, entryID, entry, suffix))
4591N/A {
4591N/A suffix.removePending(entryDN);
4643N/A return;
4591N/A }
4591N/A if(!suffix.getDN2ID().insert(null, entryDN, entryID))
4591N/A {
4591N/A suffix.removePending(entryDN);
4649N/A Message message = WARN_JEB_IMPORT_ENTRY_EXISTS.get();
4649N/A reader.rejectEntry(entry, message);
4643N/A return;
4591N/A }
4591N/A suffix.removePending(entryDN);
4591N/A processID2SC(entryID, entry, suffix);
4591N/A }
4591N/A else
4591N/A {
4591N/A processDN2ID(suffix, entryDN, entryID);
4591N/A suffix.removePending(entryDN);
4591N/A }
4643N/A }
4643N/A else
4643N/A {
4643N/A suffix.removePending(entryDN);
4643N/A entryID = oldID;
4643N/A }
4700N/A processDN2URI(suffix, oldEntry, entry);
4643N/A suffix.getID2Entry().put(null, entryID, entry);
4643N/A if(oldEntry == null)
4643N/A {
4591N/A processIndexes(suffix, entry, entryID);
4591N/A }
4643N/A else
4643N/A {
4643N/A processAllIndexes(suffix, entry, entryID);
4643N/A }
4643N/A }
4643N/A
4643N/A void
4649N/A processAllIndexes(Suffix suffix, Entry entry, EntryID entryID) throws
4643N/A DatabaseException, DirectoryException, JebException, ConfigException
4643N/A {
4649N/A
4643N/A for(Map.Entry<AttributeType, AttributeIndex> mapEntry :
4649N/A suffix.getAttrIndexMap().entrySet()) {
4649N/A AttributeType attributeType = mapEntry.getKey();
4643N/A AttributeIndex attributeIndex = mapEntry.getValue();
4643N/A Index index;
4643N/A if((index=attributeIndex.getEqualityIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.EQUALITY));
4643N/A }
4643N/A if((index=attributeIndex.getPresenceIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.PRESENCE));
4643N/A }
4643N/A if((index=attributeIndex.getSubstringIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.SUBSTRING));
4643N/A }
4643N/A if((index=attributeIndex.getOrderingIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.ORDERING));
4643N/A }
4643N/A if((index=attributeIndex.getApproximateIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
4643N/A }
4649N/A for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
4649N/A Transaction transaction = null;
4649N/A vlvIdx.addEntry(transaction, entryID, entry);
4643N/A }
4643N/A Map<String,Collection<Index>> extensibleMap =
4643N/A attributeIndex.getExtensibleIndexes();
4643N/A if(!extensibleMap.isEmpty()) {
4643N/A Collection<Index> subIndexes =
4643N/A attributeIndex.getExtensibleIndexes().get(
4643N/A EXTENSIBLE_INDEXER_ID_SUBSTRING);
4643N/A if(subIndexes != null) {
4643N/A for(Index subIndex: subIndexes) {
4649N/A processAttribute(subIndex, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
4643N/A }
4643N/A }
4643N/A Collection<Index> sharedIndexes =
4643N/A attributeIndex.getExtensibleIndexes().get(
4643N/A EXTENSIBLE_INDEXER_ID_SHARED);
4643N/A if(sharedIndexes !=null) {
4643N/A for(Index sharedIndex:sharedIndexes) {
4649N/A processAttribute(sharedIndex, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.EX_SHARED));
4643N/A }
4643N/A }
4643N/A }
4643N/A }
4643N/A }
4643N/A
4643N/A
4643N/A
4649N/A void processAttribute(Index index, Entry entry, EntryID entryID,
4643N/A IndexKey indexKey) throws DatabaseException,
4643N/A ConfigException
4643N/A {
4643N/A
4643N/A if(oldEntry != null)
4643N/A {
4643N/A deleteKeySet.clear();
4643N/A index.indexer.indexEntry(oldEntry, deleteKeySet);
4643N/A for(byte[] delKey : deleteKeySet)
4643N/A {
4643N/A processKey(index, delKey, entryID, indexComparator, indexKey, false);
4643N/A }
4643N/A }
4643N/A insertKeySet.clear();
4643N/A index.indexer.indexEntry(entry, insertKeySet);
4643N/A for(byte[] key : insertKeySet)
4643N/A {
4643N/A processKey(index, key, entryID, indexComparator, indexKey, true);
4643N/A }
4643N/A }
4643N/A }
4643N/A
4643N/A
4643N/A
4643N/A /**
4643N/A * This task processes the LDIF file during phase 1.
4643N/A */
4643N/A private class ImportTask implements Callable<Void>
4643N/A {
4643N/A
4643N/A private final
4643N/A Map<IndexKey, IndexBuffer> indexBufferMap =
4643N/A new HashMap<IndexKey, IndexBuffer>();
4643N/A private final Set<byte[]> insertKeySet = new HashSet<byte[]>();
4643N/A private final EntryInformation entryInfo = new EntryInformation();
4643N/A
4643N/A /**
4643N/A * {@inheritDoc}
4643N/A */
4643N/A public Void call() throws Exception
4643N/A {
4660N/A try
4643N/A {
4660N/A while (true)
4643N/A {
4660N/A if (importConfiguration.isCancelled())
4660N/A {
4660N/A IndexBuffer indexBuffer = IndexBuffer.createIndexBuffer(0);
4660N/A freeBufferQueue.add(indexBuffer);
4660N/A return null;
4660N/A }
4660N/A Entry entry = reader.readEntry(dnSuffixMap, entryInfo);
4660N/A
4660N/A if (entry == null)
4660N/A {
4660N/A break;
4660N/A }
4660N/A EntryID entryID = entryInfo.getEntryID();
4660N/A Suffix suffix = entryInfo.getSuffix();
4660N/A processEntry(entry, entryID, suffix);
4679N/A importCount.getAndIncrement();
4643N/A }
4660N/A flushIndexBuffers();
4660N/A closeCursors();
4643N/A }
4660N/A catch (Exception e)
4660N/A {
4660N/A Message message =
4660N/A ERR_JEB_IMPORT_LDIF_IMPORT_TASK_ERR.get(e.getMessage());
4660N/A logError(message);
4660N/A throw e;
4660N/A }
4660N/A
4591N/A return null;
4591N/A }
4591N/A
4591N/A
4643N/A void closeCursors() throws DatabaseException
4643N/A {
4643N/A if(!skipDNValidation)
4643N/A {
4643N/A for(Suffix suffix : dnSuffixMap.values())
4643N/A {
4643N/A suffix.getEntryContainer().getID2Children().closeCursor();
4643N/A suffix.getEntryContainer().getID2Subtree().closeCursor();
4643N/A }
4643N/A }
4643N/A }
4643N/A
4643N/A
4643N/A void processEntry(Entry entry, EntryID entryID, Suffix suffix)
4643N/A throws DatabaseException, ConfigException, DirectoryException,
4643N/A JebException
4643N/A
4643N/A {
4643N/A DN entryDN = entry.getDN();
4643N/A if(!skipDNValidation)
4643N/A {
4643N/A if(!processParent(entryDN, entryID, entry, suffix))
4643N/A {
4643N/A suffix.removePending(entryDN);
4643N/A return;
4643N/A }
4643N/A if(!suffix.getDN2ID().insert(null, entryDN, entryID))
4643N/A {
4643N/A suffix.removePending(entryDN);
4649N/A Message message = WARN_JEB_IMPORT_ENTRY_EXISTS.get();
4649N/A reader.rejectEntry(entry, message);
4643N/A return;
4643N/A }
4643N/A suffix.removePending(entryDN);
4643N/A processID2SC(entryID, entry, suffix);
4643N/A }
4643N/A else
4643N/A {
4643N/A processDN2ID(suffix, entryDN, entryID);
4643N/A suffix.removePending(entryDN);
4643N/A }
4700N/A processDN2URI(suffix, null, entry);
4643N/A suffix.getID2Entry().put(null, entryID, entry);
4643N/A processIndexes(suffix, entry, entryID);
4643N/A }
4643N/A
4643N/A boolean processParent(DN entryDN, EntryID entryID, Entry entry,
4591N/A Suffix suffix) throws DatabaseException
4591N/A {
4591N/A EntryID parentID = null;
4591N/A DN parentDN =
4591N/A suffix.getEntryContainer().getParentWithinBase(entryDN);
4591N/A DN2ID dn2id = suffix.getDN2ID();
4591N/A if(dn2id.get(null, entryDN, LockMode.DEFAULT) != null)
4591N/A {
4649N/A Message message = WARN_JEB_IMPORT_ENTRY_EXISTS.get();
4649N/A reader.rejectEntry(entry, message);
4591N/A return false;
4591N/A }
4591N/A
4591N/A if (parentDN != null) {
4591N/A parentID = suffix.getParentID(parentDN);
4591N/A if (parentID == null) {
4591N/A dn2id.remove(null, entryDN);
4649N/A Message message =
4591N/A ERR_JEB_IMPORT_PARENT_NOT_FOUND.get(parentDN.toString());
4649N/A reader.rejectEntry(entry, message);
4591N/A return false;
4591N/A }
4591N/A }
4591N/A ArrayList<EntryID> IDs;
4591N/A if (parentDN != null && suffix.getParentDN() != null &&
4591N/A parentDN.equals(suffix.getParentDN())) {
4591N/A IDs = new ArrayList<EntryID>(suffix.getIDs());
4591N/A IDs.set(0, entryID);
4591N/A }
4591N/A else
4591N/A {
4591N/A EntryID nodeID;
4591N/A IDs = new ArrayList<EntryID>(entryDN.getNumComponents());
4591N/A IDs.add(entryID);
4591N/A if (parentID != null)
4591N/A {
4591N/A IDs.add(parentID);
4649N/A EntryContainer entryContainer = suffix.getEntryContainer();
4649N/A for (DN dn = entryContainer.getParentWithinBase(parentDN); dn != null;
4649N/A dn = entryContainer.getParentWithinBase(dn)) {
4591N/A if((nodeID = getAncestorID(dn2id, dn)) == null) {
4591N/A return false;
4591N/A } else {
4591N/A IDs.add(nodeID);
4591N/A }
4591N/A }
4591N/A }
4591N/A }
4591N/A suffix.setParentDN(parentDN);
4591N/A suffix.setIDs(IDs);
4591N/A entry.setAttachment(IDs);
4591N/A return true;
4591N/A }
4591N/A
4643N/A void processID2SC(EntryID entryID, Entry entry, Suffix suffix)
4591N/A throws DatabaseException
4591N/A {
4591N/A Set<byte[]> childKeySet = new HashSet<byte[]>();
4649N/A Set<byte[]> subTreeKeySet = new HashSet<byte[]>();
4591N/A Index id2children = suffix.getEntryContainer().getID2Children();
4591N/A Index id2subtree = suffix.getEntryContainer().getID2Subtree();
4591N/A id2children.indexer.indexEntry(entry, childKeySet);
4649N/A id2subtree.indexer.indexEntry(entry, subTreeKeySet);
4591N/A
4591N/A DatabaseEntry dbKey = new DatabaseEntry();
4591N/A DatabaseEntry dbVal = new DatabaseEntry();
4643N/A ImportIDSet idSet = new ImportIDSet(1, id2children.getIndexEntryLimit(),
4643N/A id2children.getMaintainCount());
4643N/A idSet.addEntryID(entryID);
4591N/A id2children.insert(idSet, childKeySet, dbKey, dbVal);
4591N/A
4591N/A DatabaseEntry dbSubKey = new DatabaseEntry();
4591N/A DatabaseEntry dbSubVal = new DatabaseEntry();
4643N/A ImportIDSet idSubSet = new ImportIDSet(1, id2subtree.getIndexEntryLimit(),
4643N/A id2subtree.getMaintainCount());
4643N/A idSubSet.addEntryID(entryID);
4649N/A id2subtree.insert(idSubSet, subTreeKeySet, dbSubKey, dbSubVal);
4591N/A }
4591N/A
4643N/A EntryID getAncestorID(DN2ID dn2id, DN dn)
4591N/A throws DatabaseException
4591N/A {
4591N/A int i=0;
4591N/A EntryID nodeID = dn2id.get(null, dn, LockMode.DEFAULT);
4591N/A if(nodeID == null) {
4591N/A while((nodeID = dn2id.get(null, dn, LockMode.DEFAULT)) == null) {
4591N/A try {
4591N/A Thread.sleep(50);
4651N/A if(i == 10) {
4700N/A //Temporary messages until this code is cleaned up.
4700N/A Message message =
4700N/A Message.raw(Category.JEB, Severity.SEVERE_ERROR,
4700N/A "ancestorID check failed");
4700N/A logError(message);
4591N/A return null;
4591N/A }
4591N/A i++;
4591N/A } catch (Exception e) {
4700N/A //Temporary messages until this code is cleaned up.
4700N/A Message message =
4700N/A Message.raw(Category.JEB, Severity.SEVERE_ERROR,
4700N/A "ancestorID exception thrown");
4700N/A logError(message);
4591N/A return null;
4591N/A }
4591N/A }
4591N/A }
4591N/A return nodeID;
4591N/A }
4591N/A
4591N/A
4591N/A
4643N/A void
4649N/A processIndexes(Suffix suffix, Entry entry, EntryID entryID) throws
4591N/A DatabaseException, DirectoryException, JebException, ConfigException
4591N/A {
4591N/A for(Map.Entry<AttributeType, AttributeIndex> mapEntry :
4649N/A suffix.getAttrIndexMap().entrySet()) {
4649N/A AttributeType attributeType = mapEntry.getKey();
4649N/A if(entry.hasAttribute(attributeType)) {
4591N/A AttributeIndex attributeIndex = mapEntry.getValue();
4591N/A Index index;
4591N/A if((index=attributeIndex.getEqualityIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.EQUALITY));
4591N/A }
4591N/A if((index=attributeIndex.getPresenceIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.PRESENCE));
4591N/A }
4591N/A if((index=attributeIndex.getSubstringIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.SUBSTRING));
4591N/A }
4591N/A if((index=attributeIndex.getOrderingIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.ORDERING));
4591N/A }
4591N/A if((index=attributeIndex.getApproximateIndex()) != null) {
4649N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
4591N/A }
4649N/A for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
4649N/A Transaction transaction = null;
4649N/A vlvIdx.addEntry(transaction, entryID, entry);
4591N/A }
4591N/A Map<String,Collection<Index>> extensibleMap =
4591N/A attributeIndex.getExtensibleIndexes();
4591N/A if(!extensibleMap.isEmpty()) {
4591N/A Collection<Index> subIndexes =
4591N/A attributeIndex.getExtensibleIndexes().get(
4591N/A EXTENSIBLE_INDEXER_ID_SUBSTRING);
4591N/A if(subIndexes != null) {
4591N/A for(Index subIndex: subIndexes) {
4649N/A processAttribute(subIndex, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
4591N/A }
4591N/A }
4591N/A Collection<Index> sharedIndexes =
4591N/A attributeIndex.getExtensibleIndexes().get(
4591N/A EXTENSIBLE_INDEXER_ID_SHARED);
4591N/A if(sharedIndexes !=null) {
4591N/A for(Index sharedIndex:sharedIndexes) {
4649N/A processAttribute(sharedIndex, entry, entryID,
4765N/A new IndexKey(attributeType, ImportIndexType.EX_SHARED));
4591N/A }
4591N/A }
4591N/A }
4591N/A }
4591N/A }
4591N/A }
4591N/A
4591N/A
4591N/A
4649N/A void processAttribute(Index index, Entry entry, EntryID entryID,
4643N/A IndexKey indexKey) throws DatabaseException,
4643N/A ConfigException
4591N/A {
4591N/A insertKeySet.clear();
4591N/A index.indexer.indexEntry(entry, insertKeySet);
4591N/A for(byte[] key : insertKeySet)
4591N/A {
4643N/A processKey(index, key, entryID, indexComparator, indexKey, true);
4591N/A }
4591N/A }
4591N/A
4591N/A
4643N/A void flushIndexBuffers() throws InterruptedException,
4591N/A ExecutionException
4591N/A {
4643N/A Set<Map.Entry<IndexKey, IndexBuffer>> set = indexBufferMap.entrySet();
4643N/A for(Map.Entry<IndexKey, IndexBuffer> e : set)
4591N/A {
4643N/A IndexKey indexKey = e.getKey();
4591N/A IndexBuffer indexBuffer = e.getValue();
4765N/A ImportIndexType indexType = indexKey.getIndexType();
4765N/A if(indexType.equals(ImportIndexType.DN))
4591N/A {
4591N/A indexBuffer.setComparator(dnComparator);
4591N/A }
4591N/A else
4591N/A {
4591N/A indexBuffer.setComparator(indexComparator);
4591N/A }
4643N/A indexBuffer.setIndexKey(indexKey);
4591N/A Future<Void> future = sortService.submit(new SortTask(indexBuffer));
4591N/A future.get();
4591N/A }
4591N/A }
4591N/A
4591N/A
4643N/A int
4643N/A processKey(DatabaseContainer container, byte[] key, EntryID entryID,
4643N/A IndexBuffer.ComparatorBuffer<byte[]> comparator, IndexKey indexKey,
4643N/A boolean insert)
4643N/A throws ConfigException
4591N/A {
4591N/A IndexBuffer indexBuffer;
4643N/A if(!indexBufferMap.containsKey(indexKey))
4591N/A {
4591N/A indexBuffer = getNewIndexBuffer();
4643N/A indexBufferMap.put(indexKey, indexBuffer);
4591N/A }
4591N/A else
4591N/A {
4643N/A indexBuffer = indexBufferMap.get(indexKey);
4591N/A }
4591N/A if(!indexBuffer.isSpaceAvailable(key))
4591N/A {
4591N/A indexBuffer.setComparator(comparator);
4643N/A indexBuffer.setIndexKey(indexKey);
4591N/A sortService.submit(new SortTask(indexBuffer));
4591N/A indexBuffer = getNewIndexBuffer();
4643N/A indexBufferMap.remove(indexKey);
4643N/A indexBufferMap.put(indexKey, indexBuffer);
4591N/A }
4643N/A int id = System.identityHashCode(container);
4643N/A idContainerMap.putIfAbsent(id, container);
4643N/A indexBuffer.add(key, entryID, id, insert);
4643N/A return id;
4591N/A }
4591N/A
4591N/A
4643N/A IndexBuffer getNewIndexBuffer() throws ConfigException
4591N/A {
4649N/A IndexBuffer indexBuffer = freeBufferQueue.poll();
4591N/A if(indexBuffer.isPoison())
4591N/A {
4649N/A Message message = Message.raw(Category.JEB, Severity.SEVERE_ERROR,
4591N/A "Abort import - MPD");
4649N/A throw new ConfigException(message);
4591N/A }
4591N/A return indexBuffer;
4591N/A }
4591N/A
4591N/A
4643N/A void processDN2ID(Suffix suffix, DN dn, EntryID entryID)
4591N/A throws ConfigException
4591N/A {
4591N/A DatabaseContainer dn2id = suffix.getDN2ID();
4591N/A byte[] dnBytes = StaticUtils.getBytes(dn.toNormalizedString());
4643N/A int id = processKey(dn2id, dnBytes, entryID, dnComparator,
4765N/A new IndexKey(dnType, ImportIndexType.DN), true);
4643N/A idECMap.putIfAbsent(id, suffix.getEntryContainer());
4591N/A }
4700N/A
4700N/A void processDN2URI(Suffix suffix, Entry oldEntry, Entry newEntry)
4700N/A throws DatabaseException
4700N/A {
4700N/A DN2URI dn2uri = suffix.getDN2URI();
4700N/A if(oldEntry != null)
4700N/A {
4700N/A dn2uri.replaceEntry(null, oldEntry, newEntry);
4700N/A }
4700N/A else
4700N/A {
4700N/A dn2uri.addEntry(null, newEntry);
4700N/A }
4700N/A }
4591N/A }
4591N/A
4643N/A
4591N/A /**
4591N/A * The task reads the temporary index files and writes their results to the
4591N/A * index database.
4591N/A */
4643N/A private final class IndexWriteDBTask implements Callable<Void>
4643N/A {
4591N/A private final IndexManager indexMgr;
4591N/A private final DatabaseEntry dbKey, dbValue;
4591N/A private final int cacheSize;
4591N/A private ByteBuffer directBuffer = null;
4643N/A private final Map<Integer, DNState> dnStateMap =
4643N/A new HashMap<Integer, DNState>();
4643N/A private final Map<Integer, Index> indexMap = new HashMap<Integer, Index>();
4591N/A
4643N/A public IndexWriteDBTask(IndexManager indexMgr, ByteBuffer b, int cacheSize)
4591N/A {
4591N/A this.indexMgr = indexMgr;
4643N/A directBuffer = b;
4591N/A this.dbKey = new DatabaseEntry();
4591N/A this.dbValue = new DatabaseEntry();
4591N/A this.cacheSize = cacheSize;
4591N/A }
4591N/A
4649N/A private SortedSet<Buffer> initializeBuffers() throws IOException
4591N/A {
4591N/A int p = 0;
4591N/A int offSet = cacheSize;
4643N/A SortedSet<Buffer> bufferSet = new TreeSet<Buffer>();
4643N/A for(Buffer b : indexMgr.getBufferList())
4591N/A {
4591N/A if(directBuffer != null)
4591N/A {
4591N/A directBuffer.position(p);
4591N/A directBuffer.limit(offSet);
4591N/A ByteBuffer slice = directBuffer.slice();
4649N/A b.initializeCache(indexMgr, slice, cacheSize);
4591N/A p += cacheSize;
4591N/A offSet += cacheSize;
4591N/A }
4591N/A else
4591N/A {
4649N/A b.initializeCache(indexMgr, null, cacheSize);
4591N/A }
4591N/A bufferSet.add(b);
4591N/A }
4736N/A indexMgr.getBufferList().clear();
4643N/A return bufferSet;
4643N/A }
4643N/A
4643N/A public Void call() throws Exception
4643N/A {
4643N/A byte[] cKey = null;
4736N/A ImportIDSet cInsertIDSet = new ImportIDSet(),
4736N/A cDeleteIDSet = new ImportIDSet();
4643N/A Integer cIndexID = null;
4660N/A try
4591N/A {
4660N/A indexMgr.openIndexFile();
4660N/A SortedSet<Buffer> bufferSet = initializeBuffers();
4660N/A while(!bufferSet.isEmpty())
4591N/A {
4660N/A Buffer b;
4660N/A b = bufferSet.first();
4660N/A bufferSet.remove(b);
4660N/A if(cKey == null)
4591N/A {
4643N/A cIndexID = b.getIndexID();
4643N/A cKey = b.getKey();
4736N/A cInsertIDSet.merge(b.getInsertIDSet());
4736N/A cDeleteIDSet.merge(b.getDeleteIDSet());
4643N/A cInsertIDSet.setKey(cKey);
4643N/A cDeleteIDSet.setKey(cKey);
4591N/A }
4591N/A else
4591N/A {
4660N/A if(b.compare(cKey, cIndexID) != 0)
4660N/A {
4660N/A addToDB(cInsertIDSet, cDeleteIDSet, cIndexID);
4660N/A indexMgr.incrementKeyCount();
4660N/A cIndexID = b.getIndexID();
4660N/A cKey = b.getKey();
4736N/A cInsertIDSet.clear(true);
4736N/A cDeleteIDSet.clear(true);
4736N/A cInsertIDSet.merge(b.getInsertIDSet());
4736N/A cDeleteIDSet.merge(b.getDeleteIDSet());
4660N/A cInsertIDSet.setKey(cKey);
4660N/A cDeleteIDSet.setKey(cKey);
4660N/A }
4660N/A else
4660N/A {
4660N/A cInsertIDSet.merge(b.getInsertIDSet());
4660N/A cDeleteIDSet.merge(b.getDeleteIDSet());
4660N/A }
4660N/A }
4660N/A if(b.hasMoreData())
4660N/A {
4660N/A b.getNextRecord();
4660N/A bufferSet.add(b);
4591N/A }
4591N/A }
4660N/A if(cKey != null)
4591N/A {
4660N/A addToDB(cInsertIDSet, cDeleteIDSet, cIndexID);
4591N/A }
4660N/A cleanUP();
4591N/A }
4660N/A catch (Exception e)
4591N/A {
4660N/A Message message =
4660N/A ERR_JEB_IMPORT_LDIF_INDEX_WRITE_DB_ERR.get(indexMgr.getFileName(),
4660N/A e.getMessage());
4660N/A logError(message);
4736N/A e.printStackTrace();
4660N/A throw e;
4591N/A }
4591N/A return null;
4591N/A }
4591N/A
4591N/A
4591N/A private void cleanUP() throws DatabaseException, DirectoryException,
4591N/A IOException
4591N/A {
4643N/A if(indexMgr.isDN2ID() && skipDNValidation)
4643N/A {
4643N/A for(DNState dnState : dnStateMap.values())
4643N/A {
4643N/A dnState.flush();
4643N/A }
4643N/A Message msg = NOTE_JEB_IMPORT_LDIF_DN_CLOSE.get(indexMgr.getDNCount());
4591N/A logError(msg);
4591N/A }
4591N/A else
4591N/A {
4643N/A for(Index index : indexMap.values())
4591N/A {
4643N/A index.closeCursor();
4591N/A }
4649N/A Message message =
4649N/A NOTE_JEB_IMPORT_LDIF_INDEX_CLOSE.get(indexMgr.getFileName());
4649N/A logError(message);
4591N/A }
4591N/A indexMgr.setDone();
4591N/A indexMgr.close();
4591N/A indexMgr.deleteIndexFile();
4591N/A }
4591N/A
4643N/A private void addToDB(ImportIDSet insRec, ImportIDSet delRec, int indexID)
4643N/A throws InterruptedException, DatabaseException, DirectoryException
4643N/A {
4643N/A if(!indexMgr.isDN2ID())
4643N/A {
4643N/A Index index;
4643N/A if((delRec.size() > 0) || (!delRec.isDefined()))
4643N/A {
4643N/A dbKey.setData(delRec.getKey());
4643N/A index = (Index)idContainerMap.get(indexID);
4643N/A index.delete(dbKey, delRec, dbValue);
4643N/A if(!indexMap.containsKey(indexID))
4643N/A {
4643N/A indexMap.put(indexID, index);
4643N/A }
4643N/A }
4591N/A
4643N/A
4643N/A if((insRec.size() > 0) || (!insRec.isDefined()))
4643N/A {
4643N/A dbKey.setData(insRec.getKey());
4643N/A index = (Index)idContainerMap.get(indexID);
4643N/A index.insert(dbKey, insRec, dbValue);
4643N/A if(!indexMap.containsKey(indexID))
4643N/A {
4643N/A indexMap.put(indexID, index);
4643N/A }
4643N/A }
4643N/A }
4643N/A else if(skipDNValidation)
4643N/A {
4643N/A addDN2ID(insRec, indexID);
4643N/A }
4643N/A }
4643N/A
4643N/A private void addDN2ID(ImportIDSet record, Integer indexID)
4591N/A throws DatabaseException, DirectoryException
4591N/A {
4643N/A DNState dnState;
4643N/A if(!dnStateMap.containsKey(indexID))
4591N/A {
4643N/A dnState = new DNState(idECMap.get(indexID));
4643N/A dnStateMap.put(indexID, dnState);
4591N/A }
4643N/A else
4591N/A {
4643N/A dnState = dnStateMap.get(indexID);
4591N/A }
4643N/A
4643N/A if(!dnState.checkParent(record))
4643N/A {
4643N/A return;
4643N/A }
4643N/A dnState.writeToDB();
4591N/A }
4591N/A
4591N/A
4649N/A /**
4643N/A * This class is used to by a index DB merge thread performing DN processing
4643N/A * to keep track of the state of individual DN2ID index processing.
4643N/A */
4643N/A class DNState
4591N/A {
4736N/A private DN parentDN, lastDN;
4736N/A private EntryID parentID, lastID, entryID;
4736N/A private final DatabaseEntry DNKey, DNValue;
4643N/A private final TreeMap<DN, EntryID> parentIDMap =
4643N/A new TreeMap<DN, EntryID>();
4643N/A private final EntryContainer entryContainer;
4643N/A private final Map<byte[], ImportIDSet> id2childTree;
4643N/A private final Map<byte[], ImportIDSet> id2subtreeTree;
4736N/A private final int childLimit, subTreeLimit;
4736N/A private final boolean childDoCount, subTreeDoCount;
4643N/A
4643N/A DNState(EntryContainer entryContainer)
4591N/A {
4643N/A this.entryContainer = entryContainer;
4736N/A Comparator<byte[]> childComparator =
4736N/A entryContainer.getID2Children().getComparator();
4643N/A id2childTree = new TreeMap<byte[], ImportIDSet>(childComparator);
4736N/A childLimit = entryContainer.getID2Children().getIndexEntryLimit();
4736N/A childDoCount = entryContainer.getID2Children().getMaintainCount();
4736N/A Comparator<byte[]> subComparator =
4736N/A entryContainer.getID2Subtree().getComparator();
4736N/A subTreeLimit = entryContainer.getID2Subtree().getIndexEntryLimit();
4736N/A subTreeDoCount = entryContainer.getID2Subtree().getMaintainCount();
4643N/A id2subtreeTree = new TreeMap<byte[], ImportIDSet>(subComparator);
4736N/A DNKey = new DatabaseEntry();
4736N/A DNValue = new DatabaseEntry();
4591N/A }
4591N/A
4591N/A
4643N/A private boolean checkParent(ImportIDSet record) throws DirectoryException
4591N/A {
4736N/A DNKey.setData(record.getKey());
4643N/A byte[] v = record.toDatabase();
4643N/A long v1 = JebFormat.entryIDFromDatabase(v);
4736N/A DNValue.setData(v);
4736N/A DN dn = DN.decode(ByteString.wrap(DNKey.getData()));
4643N/A
4643N/A entryID = new EntryID(v1);
4643N/A if(parentIDMap.isEmpty())
4643N/A {
4643N/A parentIDMap.put(dn, entryID);
4643N/A return true;
4643N/A }
4643N/A else if(lastDN != null && lastDN.isAncestorOf(dn))
4591N/A {
4643N/A parentIDMap.put(lastDN, lastID);
4643N/A parentDN = lastDN;
4643N/A parentID = lastID;
4643N/A lastDN = dn;
4643N/A lastID = entryID;
4643N/A return true;
4643N/A }
4643N/A else if(parentIDMap.lastKey().isAncestorOf(dn))
4643N/A {
4643N/A parentDN = parentIDMap.lastKey();
4643N/A parentID = parentIDMap.get(parentDN);
4643N/A lastDN = dn;
4643N/A lastID = entryID;
4643N/A return true;
4591N/A }
4591N/A else
4591N/A {
4656N/A DN newParentDN = entryContainer.getParentWithinBase(dn);
4656N/A if(parentIDMap.containsKey(newParentDN))
4656N/A {
4656N/A EntryID newParentID = parentIDMap.get(newParentDN);
4656N/A DN lastDN = parentIDMap.lastKey();
4656N/A while(!newParentDN.equals(lastDN)) {
4656N/A parentIDMap.remove(lastDN);
4656N/A lastDN = parentIDMap.lastKey();
4643N/A }
4725N/A parentIDMap.put(dn, entryID);
4656N/A parentDN = newParentDN;
4656N/A parentID = newParentID;
4643N/A lastDN = dn;
4643N/A lastID = entryID;
4643N/A }
4643N/A else
4643N/A {
4649N/A Message message =
4649N/A NOTE_JEB_IMPORT_LDIF_DN_NO_PARENT.get(dn.toString());
4643N/A Entry e = new Entry(dn, null, null, null);
4649N/A reader.rejectEntry(e, message);
4643N/A return false;
4643N/A }
4591N/A }
4643N/A return true;
4591N/A }
4643N/A
4591N/A
4656N/A private void id2child(EntryID childID)
4736N/A throws DatabaseException, DirectoryException
4591N/A {
4736N/A ImportIDSet idSet;
4736N/A if(!id2childTree.containsKey(parentID.getDatabaseEntry().getData()))
4736N/A {
4736N/A idSet = new ImportIDSet(1,childLimit, childDoCount);
4736N/A id2childTree.put(parentID.getDatabaseEntry().getData(), idSet);
4736N/A }
4736N/A else
4736N/A {
4736N/A idSet = id2childTree.get(parentID.getDatabaseEntry().getData());
4736N/A }
4736N/A idSet.addEntryID(childID);
4736N/A if(id2childTree.size() > DN_STATE_CACHE_SIZE)
4736N/A {
4736N/A flushMapToDB(id2childTree, entryContainer.getID2Children(), true);
4736N/A }
4591N/A }
4591N/A
4643N/A
4736N/A private void id2SubTree(EntryID childID)
4736N/A throws DatabaseException, DirectoryException
4591N/A {
4643N/A ImportIDSet idSet;
4643N/A if(!id2subtreeTree.containsKey(parentID.getDatabaseEntry().getData()))
4643N/A {
4736N/A idSet = new ImportIDSet(1, subTreeLimit, subTreeDoCount);
4643N/A id2subtreeTree.put(parentID.getDatabaseEntry().getData(), idSet);
4591N/A }
4591N/A else
4591N/A {
4643N/A idSet = id2subtreeTree.get(parentID.getDatabaseEntry().getData());
4643N/A }
4643N/A idSet.addEntryID(childID);
4643N/A for (DN dn = entryContainer.getParentWithinBase(parentDN); dn != null;
4643N/A dn = entryContainer.getParentWithinBase(dn))
4643N/A {
4643N/A EntryID nodeID = parentIDMap.get(dn);
4643N/A if(!id2subtreeTree.containsKey(nodeID.getDatabaseEntry().getData()))
4643N/A {
4736N/A idSet = new ImportIDSet(1, subTreeLimit, subTreeDoCount);
4643N/A id2subtreeTree.put(nodeID.getDatabaseEntry().getData(), idSet);
4643N/A }
4643N/A else
4643N/A {
4643N/A idSet = id2subtreeTree.get(nodeID.getDatabaseEntry().getData());
4643N/A }
4643N/A idSet.addEntryID(childID);
4591N/A }
4736N/A if (id2subtreeTree.size() > DN_STATE_CACHE_SIZE)
4736N/A {
4736N/A flushMapToDB(id2subtreeTree, entryContainer.getID2Subtree(), true);
4736N/A }
4591N/A }
4643N/A
4591N/A
4736N/A public void writeToDB() throws DatabaseException, DirectoryException
4591N/A {
4736N/A entryContainer.getDN2ID().putRaw(null, DNKey, DNValue);
4736N/A indexMgr.addTotDNCount(1);
4736N/A if(parentDN != null)
4736N/A {
4736N/A id2child(entryID);
4736N/A id2SubTree(entryID);
4736N/A }
4591N/A }
4591N/A
4736N/A private void flushMapToDB(Map<byte[], ImportIDSet> map, Index index,
4736N/A boolean clearMap)
4736N/A throws DatabaseException, DirectoryException
4736N/A {
4736N/A for(Map.Entry<byte[], ImportIDSet> e : map.entrySet())
4736N/A {
4736N/A byte[] key = e.getKey();
4736N/A ImportIDSet idSet = e.getValue();
4736N/A DNKey.setData(key);
4736N/A index.insert(DNKey, idSet, DNValue);
4736N/A }
4736N/A index.closeCursor();
4736N/A if(clearMap)
4736N/A {
4736N/A map.clear();
4736N/A }
4736N/A }
4591N/A
4643N/A public void flush() throws DatabaseException, DirectoryException
4643N/A {
4736N/A flushMapToDB(id2childTree, entryContainer.getID2Children(), false);
4736N/A flushMapToDB(id2subtreeTree, entryContainer.getID2Subtree(), false);
4643N/A }
4591N/A }
4591N/A }
4591N/A
4591N/A
4591N/A /**
4591N/A * This task writes the temporary index files using the sorted buffers read
4591N/A * from a blocking queue.
4591N/A */
4591N/A private final class IndexFileWriterTask implements Runnable
4591N/A {
4591N/A private final IndexManager indexMgr;
4649N/A private final BlockingQueue<IndexBuffer> queue;
4643N/A private final ByteArrayOutputStream insetByteStream =
4643N/A new ByteArrayOutputStream(2 * bufferSize);
4643N/A private final ByteArrayOutputStream deleteByteStream =
4591N/A new ByteArrayOutputStream(2 * bufferSize);
4591N/A private final DataOutputStream dataStream;
4649N/A private long bufferCount = 0;
4591N/A private final File file;
4591N/A private final SortedSet<IndexBuffer> indexSortedSet;
4591N/A private boolean poisonSeen = false;
4591N/A
4649N/A public IndexFileWriterTask(BlockingQueue<IndexBuffer> queue,
4591N/A IndexManager indexMgr) throws FileNotFoundException
4591N/A {
4649N/A this.queue = queue;
4591N/A file = indexMgr.getFile();
4591N/A this.indexMgr = indexMgr;
4591N/A BufferedOutputStream bufferedStream =
4591N/A new BufferedOutputStream(new FileOutputStream(file), 2 * MB);
4591N/A dataStream = new DataOutputStream(bufferedStream);
4591N/A indexSortedSet = new TreeSet<IndexBuffer>();
4591N/A }
4591N/A
4591N/A
4591N/A public void run()
4591N/A {
4591N/A long offset = 0;
4591N/A List<IndexBuffer> l = new LinkedList<IndexBuffer>();
4591N/A try {
4591N/A while(true)
4591N/A {
4649N/A IndexBuffer indexBuffer = queue.poll();
4591N/A if(indexBuffer != null)
4591N/A {
4591N/A long beginOffset = offset;
4649N/A long bufferLen;
4649N/A if(!queue.isEmpty())
4591N/A {
4649N/A queue.drainTo(l, DRAIN_TO);
4591N/A l.add(indexBuffer);
4649N/A bufferLen = writeIndexBuffers(l);
4591N/A for(IndexBuffer id : l)
4591N/A {
4591N/A id.reset();
4591N/A }
4649N/A freeBufferQueue.addAll(l);
4591N/A l.clear();
4591N/A }
4591N/A else
4591N/A {
4591N/A if(indexBuffer.isPoison())
4591N/A {
4591N/A break;
4591N/A }
4649N/A bufferLen = writeIndexBuffer(indexBuffer);
4591N/A indexBuffer.reset();
4649N/A freeBufferQueue.add(indexBuffer);
4644N/A }
4649N/A offset += bufferLen;
4649N/A indexMgr.addBuffer(new Buffer(beginOffset, offset, bufferCount));
4649N/A bufferCount++;
4649N/A Importer.this.bufferCount.incrementAndGet();
4643N/A if(poisonSeen)
4643N/A {
4643N/A break;
4643N/A }
4591N/A }
4591N/A }
4591N/A dataStream.close();
4591N/A indexMgr.setFileLength();
4591N/A }
4660N/A catch (IOException e)
4660N/A {
4649N/A Message message =
4591N/A ERR_JEB_IMPORT_LDIF_INDEX_FILEWRITER_ERR.get(file.getName(),
4643N/A e.getMessage());
4649N/A logError(message);
4591N/A }
4591N/A }
4591N/A
4591N/A
4591N/A private long writeIndexBuffer(IndexBuffer indexBuffer) throws IOException
4591N/A {
4649N/A int numberKeys = indexBuffer.getNumberKeys();
4649N/A indexBuffer.setPosition(-1);
4649N/A long bufferLen = 0;
4643N/A insetByteStream.reset();
4643N/A deleteByteStream.reset();
4649N/A for(int i = 0; i < numberKeys; i++)
4591N/A {
4649N/A if(indexBuffer.getPosition() == -1)
4591N/A {
4649N/A indexBuffer.setPosition(i);
4643N/A if(indexBuffer.isInsert(i))
4643N/A {
4643N/A insetByteStream.write(indexBuffer.getIDBytes(i));
4643N/A }
4643N/A else
4643N/A {
4643N/A deleteByteStream.write(indexBuffer.getIDBytes(i));
4643N/A }
4591N/A continue;
4591N/A }
4591N/A if(!indexBuffer.compare(i))
4591N/A {
4649N/A bufferLen += indexBuffer.writeRecord(insetByteStream,
4649N/A deleteByteStream, dataStream);
4649N/A indexBuffer.setPosition(i);
4643N/A insetByteStream.reset();
4643N/A deleteByteStream.reset();
4591N/A }
4643N/A if(indexBuffer.isInsert(i))
4643N/A {
4643N/A insetByteStream.write(indexBuffer.getIDBytes(i));
4643N/A }
4643N/A else
4643N/A {
4643N/A deleteByteStream.write(indexBuffer.getIDBytes(i));
4643N/A }
4591N/A }
4649N/A if(indexBuffer.getPosition() != -1)
4591N/A {
4649N/A bufferLen += indexBuffer.writeRecord(insetByteStream, deleteByteStream,
4643N/A dataStream);
4591N/A }
4649N/A return bufferLen;
4591N/A }
4591N/A
4591N/A
4591N/A private long writeIndexBuffers(List<IndexBuffer> buffers)
4591N/A throws IOException
4591N/A {
4591N/A long id = 0;
4649N/A long bufferLen = 0;
4643N/A insetByteStream.reset();
4643N/A deleteByteStream.reset();
4591N/A for(IndexBuffer b : buffers)
4591N/A {
4591N/A if(b.isPoison())
4591N/A {
4591N/A poisonSeen = true;
4591N/A }
4591N/A else
4591N/A {
4649N/A b.setPosition(0);
4591N/A b.setID(id++);
4591N/A indexSortedSet.add(b);
4591N/A }
4591N/A }
4591N/A byte[] saveKey = null;
4643N/A int saveIndexID = 0;
4591N/A while(!indexSortedSet.isEmpty())
4591N/A {
4591N/A IndexBuffer b = indexSortedSet.first();
4591N/A indexSortedSet.remove(b);
4591N/A if(saveKey == null)
4591N/A {
4643N/A saveKey = b.getKeyBytes();
4643N/A saveIndexID = b.getIndexID();
4649N/A if(b.isInsert(b.getPosition()))
4643N/A {
4649N/A insetByteStream.write(b.getIDBytes(b.getPosition()));
4643N/A }
4643N/A else
4643N/A {
4649N/A deleteByteStream.write(b.getIDBytes(b.getPosition()));
4643N/A }
4591N/A }
4591N/A else
4591N/A {
4643N/A if(!b.compare(saveKey, saveIndexID))
4591N/A {
4649N/A bufferLen += IndexBuffer.writeRecord(saveKey, saveIndexID,
4643N/A insetByteStream, deleteByteStream, dataStream);
4643N/A insetByteStream.reset();
4643N/A deleteByteStream.reset();
4643N/A saveKey = b.getKeyBytes();
4643N/A saveIndexID = b.getIndexID();
4649N/A if(b.isInsert(b.getPosition()))
4643N/A {
4649N/A insetByteStream.write(b.getIDBytes(b.getPosition()));
4643N/A }
4643N/A else
4643N/A {
4649N/A deleteByteStream.write(b.getIDBytes(b.getPosition()));
4643N/A }
4591N/A }
4591N/A else
4591N/A {
4649N/A if(b.isInsert(b.getPosition()))
4643N/A {
4649N/A insetByteStream.write(b.getIDBytes(b.getPosition()));
4643N/A }
4643N/A else
4643N/A {
4649N/A deleteByteStream.write(b.getIDBytes(b.getPosition()));
4643N/A }
4591N/A }
4591N/A }
4591N/A if(b.hasMoreData())
4591N/A {
4591N/A b.getNextRecord();
4591N/A indexSortedSet.add(b);
4591N/A }
4591N/A }
4591N/A if(saveKey != null)
4591N/A {
4649N/A bufferLen += IndexBuffer.writeRecord(saveKey, saveIndexID,
4643N/A insetByteStream, deleteByteStream, dataStream);
4591N/A }
4649N/A return bufferLen;
4591N/A }
4591N/A }
4591N/A
4591N/A /**
4591N/A * This task main function is to sort the index buffers given to it from
4591N/A * the import tasks reading the LDIF file. It will also create a index
4591N/A * file writer task and corresponding queue if needed. The sorted index
4591N/A * buffers are put on the index file writer queues for writing to a temporary
4591N/A * file.
4591N/A */
4591N/A private final class SortTask implements Callable<Void>
4591N/A {
4591N/A
4591N/A private final IndexBuffer indexBuffer;
4591N/A
4591N/A public SortTask(IndexBuffer indexBuffer)
4591N/A {
4591N/A this.indexBuffer = indexBuffer;
4591N/A }
4591N/A
4591N/A /**
4591N/A * {@inheritDoc}
4591N/A */
4591N/A public Void call() throws Exception
4591N/A {
4765N/A if (importConfiguration != null &&
4765N/A importConfiguration.isCancelled())
4591N/A {
4591N/A return null;
4591N/A }
4591N/A indexBuffer.sort();
4643N/A if(indexKeyQueMap.containsKey(indexBuffer.getIndexKey())) {
4591N/A BlockingQueue<IndexBuffer> q =
4643N/A indexKeyQueMap.get(indexBuffer.getIndexKey());
4591N/A q.add(indexBuffer);
4591N/A }
4591N/A else
4591N/A {
4643N/A createIndexWriterTask(indexBuffer.getIndexKey());
4643N/A BlockingQueue<IndexBuffer> q =
4643N/A indexKeyQueMap.get(indexBuffer.getIndexKey());
4591N/A q.add(indexBuffer);
4591N/A }
4591N/A return null;
4591N/A }
4591N/A
4643N/A private void createIndexWriterTask(IndexKey indexKey)
4643N/A throws FileNotFoundException
4591N/A {
4649N/A boolean isDN = false;
4643N/A synchronized(synObj)
4643N/A {
4643N/A if(indexKeyQueMap.containsKey(indexKey))
4591N/A {
4591N/A return;
4591N/A }
4765N/A if(indexKey.getIndexType().equals(ImportIndexType.DN))
4591N/A {
4649N/A isDN = true;
4591N/A }
4649N/A IndexManager indexMgr = new IndexManager(indexKey.getName(), isDN);
4643N/A indexMgrList.add(indexMgr);
4591N/A BlockingQueue<IndexBuffer> newQue =
4643N/A new ArrayBlockingQueue<IndexBuffer>(indexBufferCount);
4591N/A IndexFileWriterTask indexWriter =
4591N/A new IndexFileWriterTask(newQue, indexMgr);
4591N/A indexWriterList.add(indexWriter);
4591N/A indexWriterFutures.add(indexProcessService.submit(indexWriter));
4643N/A indexKeyQueMap.put(indexKey, newQue);
4591N/A }
4591N/A }
4591N/A }
4591N/A
4591N/A /**
4591N/A * The buffer class is used to process a buffer from the temporary index files
4591N/A * during phase 2 processing.
4591N/A */
4591N/A private final class Buffer implements Comparable<Buffer>
4591N/A {
4591N/A private IndexManager indexMgr;
4591N/A private final long begin, end, id;
4591N/A private long offset;
4591N/A private ByteBuffer cache;
4643N/A private int keyLen, idLen, limit;
4591N/A private byte[] key;
4736N/A private ImportIDSet insertIDSet = null, deleteIDSet = null;
4643N/A private Integer indexID = null;
4643N/A private boolean doCount;
4643N/A private Comparator<byte[]> comparator;
4591N/A
4591N/A
4591N/A public Buffer(long begin, long end, long id)
4591N/A {
4591N/A this.begin = begin;
4591N/A this.end = end;
4591N/A this.offset = 0;
4591N/A this.id = id;
4591N/A }
4591N/A
4591N/A
4649N/A private void initializeCache(IndexManager indexMgr, ByteBuffer b,
4591N/A long cacheSize) throws IOException
4591N/A {
4591N/A this.indexMgr = indexMgr;
4591N/A if(b == null)
4591N/A {
4591N/A cache = ByteBuffer.allocate((int)cacheSize);
4591N/A }
4591N/A else
4591N/A {
4591N/A cache = b;
4591N/A }
4591N/A loadCache();
4591N/A cache.flip();
4591N/A }
4591N/A
4591N/A
4591N/A private void loadCache() throws IOException
4591N/A {
4591N/A FileChannel fileChannel = indexMgr.getChannel();
4591N/A fileChannel.position(begin + offset);
4591N/A long leftToRead = end - (begin + offset);
4591N/A long bytesToRead;
4591N/A if(leftToRead < cache.remaining())
4591N/A {
4649N/A cache.limit((int) (cache.position() + leftToRead));
4591N/A bytesToRead = (int)leftToRead;
4591N/A }
4591N/A else
4591N/A {
4591N/A bytesToRead = Math.min((end - offset),cache.remaining());
4591N/A }
4591N/A int bytesRead = 0;
4591N/A while(bytesRead < bytesToRead)
4591N/A {
4591N/A bytesRead += fileChannel.read(cache);
4591N/A }
4591N/A offset += bytesRead;
4591N/A indexMgr.addBytesRead(bytesRead);
4591N/A }
4591N/A
4649N/A public boolean hasMoreData() throws IOException
4591N/A {
4649N/A boolean ret = ((begin + offset) >= end) ? true: false;
4649N/A if(cache.remaining() == 0 && ret)
4649N/A {
4649N/A return false;
4649N/A }
4649N/A else
4649N/A {
4649N/A return true;
4649N/A }
4591N/A }
4591N/A
4591N/A public byte[] getKey()
4591N/A {
4591N/A return key;
4591N/A }
4591N/A
4643N/A public ImportIDSet getInsertIDSet()
4591N/A {
4643N/A return insertIDSet;
4643N/A }
4643N/A
4643N/A public ImportIDSet getDeleteIDSet()
4643N/A {
4643N/A return deleteIDSet;
4591N/A }
4591N/A
4649N/A public long getBufferID()
4591N/A {
4591N/A return id;
4591N/A }
4591N/A
4643N/A public Integer getIndexID()
4643N/A {
4643N/A if(indexID == null)
4643N/A {
4643N/A try {
4643N/A getNextRecord();
4643N/A } catch(IOException ex) {
4643N/A System.out.println("MPD need some error message");
4643N/A }
4643N/A }
4643N/A return indexID;
4643N/A }
4643N/A
4591N/A public void getNextRecord() throws IOException
4591N/A {
4643N/A getNextIndexID();
4649N/A getContainerParameters();
4591N/A getNextKey();
4643N/A getNextIDSet(true); //get insert ids
4643N/A getNextIDSet(false); //get delete ids
4643N/A }
4643N/A
4649N/A private void getContainerParameters()
4643N/A {
4643N/A limit = 1;
4643N/A doCount = false;
4643N/A if(!indexMgr.isDN2ID())
4643N/A {
4643N/A Index index = (Index) idContainerMap.get(indexID);
4643N/A limit = index.getIndexEntryLimit();
4643N/A doCount = index.getMaintainCount();
4643N/A comparator = index.getComparator();
4736N/A if(insertIDSet == null)
4736N/A {
4736N/A insertIDSet = new ImportIDSet(128, limit, doCount);
4736N/A deleteIDSet = new ImportIDSet(128, limit, doCount);
4736N/A }
4643N/A }
4643N/A else
4643N/A {
4649N/A comparator = ((DN2ID) idContainerMap.get(indexID)).getComparator();
4736N/A if(insertIDSet == null)
4736N/A {
4736N/A insertIDSet = new ImportIDSet(1, limit, doCount);
4736N/A deleteIDSet = new ImportIDSet(1, limit, doCount);
4736N/A }
4643N/A }
4591N/A }
4591N/A
4591N/A private int getInt() throws IOException
4591N/A {
4591N/A ensureData(4);
4591N/A return cache.getInt();
4591N/A }
4591N/A
4591N/A private long getLong() throws IOException
4591N/A {
4591N/A ensureData(8);
4591N/A return cache.getLong();
4591N/A }
4591N/A
4591N/A private void getBytes(byte[] b) throws IOException
4591N/A {
4591N/A ensureData(b.length);
4591N/A cache.get(b);
4591N/A }
4591N/A
4643N/A private void getNextIndexID() throws IOException, BufferUnderflowException
4643N/A {
4649N/A indexID = getInt();
4643N/A }
4643N/A
4591N/A private void getNextKey() throws IOException, BufferUnderflowException
4591N/A {
4591N/A keyLen = getInt();
4591N/A key = new byte[keyLen];
4643N/A getBytes(key);
4591N/A }
4591N/A
4643N/A private void getNextIDSet(boolean insert)
4643N/A throws IOException, BufferUnderflowException
4591N/A {
4591N/A idLen = getInt();
4591N/A int idCount = idLen/8;
4643N/A
4643N/A if(insert)
4643N/A {
4736N/A insertIDSet.clear(false);
4643N/A }
4643N/A else
4643N/A {
4736N/A deleteIDSet.clear(false);
4643N/A }
4591N/A for(int i = 0; i < idCount; i++)
4591N/A {
4591N/A long l = getLong();
4643N/A if(insert)
4643N/A {
4643N/A insertIDSet.addEntryID(l);
4643N/A }
4643N/A else
4643N/A {
4643N/A deleteIDSet.addEntryID(l);
4643N/A }
4591N/A }
4591N/A }
4591N/A
4591N/A
4591N/A private void ensureData(int len) throws IOException
4591N/A {
4591N/A if(cache.remaining() == 0)
4591N/A {
4591N/A cache.clear();
4591N/A loadCache();
4591N/A cache.flip();
4591N/A }
4591N/A else if(cache.remaining() < len)
4591N/A {
4591N/A cache.compact();
4591N/A loadCache();
4591N/A cache.flip();
4591N/A }
4591N/A }
4591N/A
4643N/A
4643N/A private int compare(byte[] cKey, Integer cIndexID)
4643N/A {
4643N/A
4649N/A int returnCode;
4643N/A if(key == null)
4643N/A {
4643N/A getIndexID();
4643N/A }
4643N/A if(comparator.compare(key, cKey) != 0) {
4649N/A returnCode = 1;
4643N/A }
4643N/A else
4643N/A {
4649N/A returnCode = (indexID.intValue() == cIndexID.intValue()) ? 0 : 1;
4643N/A }
4649N/A return returnCode;
4643N/A }
4643N/A
4643N/A
4643N/A
4591N/A public int compareTo(Buffer o) {
4643N/A //used in remove.
4591N/A if(this.equals(o))
4591N/A {
4591N/A return 0;
4591N/A }
4643N/A if(key == null) {
4643N/A getIndexID();
4643N/A }
4643N/A if(o.getKey() == null)
4643N/A {
4643N/A o.getIndexID();
4643N/A }
4649N/A int returnCode = comparator.compare(key, o.getKey());
4649N/A if(returnCode == 0)
4591N/A {
4643N/A if(indexID.intValue() == o.getIndexID().intValue())
4591N/A {
4643N/A if(insertIDSet.isDefined())
4643N/A {
4649N/A returnCode = -1;
4643N/A }
4643N/A else if(o.getInsertIDSet().isDefined())
4643N/A {
4649N/A returnCode = 1;
4643N/A }
4643N/A else if(insertIDSet.size() == o.getInsertIDSet().size())
4643N/A {
4649N/A returnCode = id > o.getBufferID() ? 1 : -1;
4643N/A }
4643N/A else
4643N/A {
4649N/A returnCode = insertIDSet.size() - o.getInsertIDSet().size();
4643N/A }
4591N/A }
4649N/A else if(indexID > o.getIndexID())
4591N/A {
4649N/A returnCode = 1;
4591N/A }
4591N/A else
4591N/A {
4649N/A returnCode = -1;
4591N/A }
4591N/A }
4649N/A return returnCode;
4591N/A }
4591N/A }
4591N/A
4591N/A /**
4591N/A * The index manager class is used to carry information about index processing
4591N/A * from phase 1 to phase 2.
4591N/A */
4591N/A private final class IndexManager
4591N/A {
4591N/A private final File file;
4649N/A private RandomAccessFile rFile = null;
4591N/A private final List<Buffer> bufferList = new LinkedList<Buffer>();
4591N/A private long fileLength, bytesRead = 0;
4591N/A private boolean done = false;
4591N/A private long totalDNS;
4591N/A private AtomicInteger keyCount = new AtomicInteger(0);
4649N/A private final String fileName;
4649N/A private final boolean isDN;
4591N/A
4649N/A public IndexManager(String fileName, boolean isDN)
4591N/A {
4649N/A file = new File(tempDir, fileName);
4649N/A this.fileName = fileName;
4649N/A this.isDN = isDN;
4591N/A }
4591N/A
4649N/A public void openIndexFile() throws FileNotFoundException
4591N/A {
4649N/A rFile = new RandomAccessFile(file, "r");
4591N/A }
4591N/A
4591N/A public FileChannel getChannel()
4591N/A {
4649N/A return rFile.getChannel();
4591N/A }
4591N/A
4591N/A public void addBuffer(Buffer o)
4591N/A {
4591N/A this.bufferList.add(o);
4591N/A }
4591N/A
4591N/A public List<Buffer> getBufferList()
4591N/A {
4591N/A return bufferList;
4591N/A }
4591N/A
4591N/A public File getFile()
4591N/A {
4591N/A return file;
4591N/A }
4591N/A
4649N/A public boolean deleteIndexFile()
4591N/A {
4649N/A return file.delete();
4591N/A }
4591N/A
4591N/A public void close() throws IOException
4591N/A {
4649N/A rFile.close();
4591N/A }
4591N/A
4591N/A public void setFileLength()
4591N/A {
4591N/A this.fileLength = file.length();
4591N/A }
4591N/A
4591N/A public void addBytesRead(int bytesRead)
4591N/A {
4591N/A this.bytesRead += bytesRead;
4591N/A }
4591N/A
4591N/A public void setDone()
4591N/A {
4591N/A this.done = true;
4591N/A }
4591N/A
4591N/A public void addTotDNCount(int delta)
4591N/A {
4591N/A this.totalDNS += delta;
4591N/A }
4591N/A
4591N/A
4643N/A public long getDNCount()
4591N/A {
4591N/A return totalDNS;
4591N/A }
4591N/A
4643N/A public boolean isDN2ID()
4643N/A {
4649N/A return isDN;
4643N/A }
4591N/A
4591N/A public void printStats(long deltaTime)
4591N/A {
4591N/A if(!done)
4591N/A {
4591N/A float rate = 1000f * keyCount.getAndSet(0) / deltaTime;
4649N/A Message message = NOTE_JEB_IMPORT_LDIF_PHASE_TWO_REPORT.get(fileName,
4591N/A (fileLength - bytesRead), rate);
4649N/A logError(message);
4591N/A }
4591N/A }
4591N/A
4649N/A public void incrementKeyCount()
4591N/A {
4591N/A keyCount.incrementAndGet();
4591N/A }
4643N/A
4649N/A public String getFileName()
4643N/A {
4649N/A return fileName;
4643N/A }
4591N/A }
4591N/A
4765N/A
4765N/A /**
4765N/A * The rebuild manager handles all rebuild index related tasks.
4765N/A */
4765N/A class RebuildManager extends ImportTask {
4765N/A
4765N/A private final RebuildConfig rebuildConfig;
4765N/A private final LocalDBBackendCfg cfg;
4765N/A private final Map<IndexKey, Index> indexMap =
4765N/A new LinkedHashMap<IndexKey, Index>();
4765N/A private final Map<IndexKey, Collection<Index>> extensibleIndexMap =
4765N/A new LinkedHashMap<IndexKey, Collection<Index>>();
4765N/A private DN2ID dn2id = null;
4765N/A private DN2URI dn2uri = null;
4765N/A private long totalEntries =0;
4765N/A private final AtomicLong entriesProcessed = new AtomicLong(0);
4765N/A private Suffix suffix = null;
4765N/A private final boolean rebuildAll;
4765N/A private EntryContainer ec;
4765N/A
4765N/A
4765N/A /**
4765N/A * Create an instance of the rebuild index manager using the specified
4765N/A * parameters.
4765N/A *
4765N/A * @param rebuildConfig The rebuild configuration to use.
4765N/A * @param cfg The local DB configuration to use.
4765N/A */
4765N/A public RebuildManager(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg)
4765N/A {
4765N/A this.rebuildConfig = rebuildConfig;
4765N/A this.cfg = cfg;
4765N/A this.rebuildAll = rebuildConfig.isRebuildAll();
4765N/A }
4765N/A
4765N/A /**
4765N/A * Initialize a rebuild manager to start rebuilding indexes.
4765N/A *
4765N/A * @throws ConfigException If an configuration error occurred.
4765N/A * @throws InitializationException If an initialization error occurred.
4765N/A */
4765N/A public void initialize() throws ConfigException, InitializationException
4765N/A {
4765N/A ec = rootContainer.getEntryContainer(rebuildConfig.getBaseDN());
4765N/A suffix = Suffix.createSuffixContext(ec, null, null, null);
4765N/A if(suffix == null)
4765N/A {
4765N/A Message msg = ERR_JEB_REBUILD_SUFFIX_ERROR.get(rebuildConfig.
4765N/A getBaseDN().toString());
4765N/A throw new InitializationException(msg);
4765N/A }
4765N/A }
4765N/A
4765N/A /**
4765N/A * Print start message.
4765N/A *
4765N/A * @throws DatabaseException If an database error occurred.
4765N/A */
4765N/A public void printStartMessage() throws DatabaseException
4765N/A {
4765N/A StringBuilder sb = new StringBuilder();
4765N/A List<String> rebuildList = rebuildConfig.getRebuildList();
4765N/A for(String index : rebuildList)
4765N/A {
4765N/A if(sb.length() > 0)
4765N/A {
4765N/A sb.append(", ");
4765N/A }
4765N/A sb.append(index);
4765N/A }
4765N/A totalEntries = suffix.getID2Entry().getRecordCount();
4765N/A Message message = NOTE_JEB_REBUILD_START.get(sb.toString(), totalEntries);
4765N/A if(rebuildAll) {
4765N/A message = NOTE_JEB_REBUILD_ALL_START.get(totalEntries);
4765N/A }
4765N/A logError(message);
4765N/A }
4765N/A
4765N/A /**
4765N/A * Print stop message.
4765N/A *
4765N/A * @param startTime The time the rebuild started.
4765N/A */
4765N/A public void printStopMessage(long startTime)
4765N/A {
4765N/A long finishTime = System.currentTimeMillis();
4765N/A long totalTime = (finishTime - startTime);
4765N/A float rate = 0;
4765N/A if (totalTime > 0)
4765N/A {
4765N/A rate = 1000f* entriesProcessed.get() / totalTime;
4765N/A }
4765N/A Message message =
4765N/A NOTE_JEB_REBUILD_FINAL_STATUS.get(entriesProcessed.get(),
4765N/A totalTime/1000, rate);
4765N/A logError(message);
4765N/A }
4765N/A
4765N/A
4765N/A /**
4765N/A * {@inheritDoc}
4765N/A */
4765N/A public Void call() throws Exception
4765N/A {
4765N/A ID2Entry id2entry = ec.getID2Entry();
4765N/A Cursor cursor = id2entry.openCursor(null, CursorConfig.READ_COMMITTED);
4765N/A DatabaseEntry key = new DatabaseEntry();
4765N/A DatabaseEntry data = new DatabaseEntry();
4765N/A LockMode lockMode = LockMode.DEFAULT;
4765N/A OperationStatus status;
4765N/A try {
4765N/A for (status = cursor.getFirst(key, data, lockMode);
4765N/A status == OperationStatus.SUCCESS;
4765N/A status = cursor.getNext(key, data, lockMode))
4765N/A {
4765N/A EntryID entryID = new EntryID(key);
4765N/A Entry entry = ID2Entry.entryFromDatabase(
4765N/A ByteString.wrap(data.getData()),
4765N/A ec.getRootContainer().getCompressedSchema());
4765N/A processEntry(entry, entryID);
4765N/A entriesProcessed.getAndIncrement();
4765N/A }
4765N/A flushIndexBuffers();
4765N/A cursor.close();
4765N/A } catch (Exception e) {
4765N/A e.printStackTrace();
4765N/A }
4765N/A return null;
4765N/A }
4765N/A
4765N/A /**
4765N/A * Perform the index rebuild.
4765N/A *
4765N/A * @throws DatabaseException If an database error occurred.
4765N/A * @throws InterruptedException If an interrupted error occurred.
4765N/A * @throws ExecutionException If an Excecution error occurred.
4765N/A * @throws JebException If an JEB error occurred.
4765N/A */
4765N/A public void rebuldIndexes() throws DatabaseException, InterruptedException,
4765N/A ExecutionException, JebException
4765N/A {
4765N/A processPhaseOne();
4765N/A processPhaseTwo();
4765N/A setIndexesTrusted();
4765N/A }
4765N/A
4765N/A private void setIndexesTrusted() throws JebException
4765N/A {
4765N/A try {
4765N/A suffix.setIndexesTrusted();
4765N/A }
4765N/A catch (DatabaseException ex)
4765N/A {
4765N/A Message message =
4765N/A NOTE_JEB_IMPORT_LDIF_TRUSTED_FAILED.get(ex.getMessage());
4765N/A throw new JebException(message);
4765N/A }
4765N/A }
4765N/A
4765N/A private void processPhaseOne() throws DatabaseException,
4765N/A InterruptedException, ExecutionException {
4765N/A if(rebuildAll)
4765N/A {
4765N/A clearAllIndexes();
4765N/A }
4765N/A else
4765N/A {
4765N/A clearRebuildListIndexes();
4765N/A }
4765N/A initializeIndexBuffers(threadCount);
4765N/A RBFirstPhaseProgressTask progressTask = new RBFirstPhaseProgressTask();
4765N/A Timer timer = new Timer();
4765N/A timer.scheduleAtFixedRate(progressTask, TIMER_INTERVAL, TIMER_INTERVAL);
4765N/A indexProcessService = Executors.newFixedThreadPool(2 * indexCount);
4765N/A sortService = Executors.newFixedThreadPool(threadCount);
4765N/A ExecutorService execService = Executors.newFixedThreadPool(threadCount);
4765N/A List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(threadCount);
4765N/A for (int i = 0; i < threadCount; i++)
4765N/A {
4765N/A tasks.add(this);
4765N/A }
4765N/A List<Future<Void>> results = execService.invokeAll(tasks);
4765N/A for (Future<Void> result : results) {
4765N/A if(!result.isDone()) {
4765N/A result.get();
4765N/A }
4765N/A }
4765N/A stopIndexWriterTasks();
4765N/A for (Future<?> result : indexWriterFutures)
4765N/A {
4765N/A if(!result.isDone()) {
4765N/A result.get();
4765N/A }
4765N/A }
4765N/A tasks.clear();
4765N/A results.clear();
4765N/A execService.shutdown();
4765N/A freeBufferQueue.clear();
4765N/A sortService.shutdown();
4765N/A timer.cancel();
4765N/A }
4765N/A
4765N/A
4765N/A private void processPhaseTwo() throws InterruptedException
4765N/A {
4765N/A SecondPhaseProgressTask progress2Task =
4765N/A new SecondPhaseProgressTask(indexMgrList, entriesProcessed.get());
4765N/A Timer timer2 = new Timer();
4765N/A timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
4765N/A processIndexFiles();
4765N/A timer2.cancel();
4765N/A }
4765N/A
4765N/A private int getIndexCount() throws ConfigException, JebException
4765N/A {
4765N/A int indexCount;
4765N/A if(!rebuildAll)
4765N/A {
4765N/A indexCount = getRebuildListIndexCount(cfg);
4765N/A }
4765N/A else
4765N/A {
4765N/A indexCount = getAllIndexesCount(cfg);
4765N/A }
4765N/A return indexCount;
4765N/A }
4765N/A
4765N/A private int getAllIndexesCount(LocalDBBackendCfg cfg)
4765N/A {
4765N/A int indexCount = cfg.listLocalDBIndexes().length;
4765N/A indexCount += cfg.listLocalDBVLVIndexes().length;
4765N/A indexCount += 4;
4765N/A return indexCount;
4765N/A }
4765N/A
4765N/A private int getRebuildListIndexCount(LocalDBBackendCfg cfg)
4765N/A throws JebException, ConfigException
4765N/A {
4765N/A int indexCount = 0;
4765N/A List<String> rebuildList = rebuildConfig.getRebuildList();
4765N/A if(!rebuildList.isEmpty())
4765N/A {
4765N/A for (String index : rebuildList)
4765N/A {
4765N/A String lowerName = index.toLowerCase();
4765N/A if (lowerName.equals("dn2id"))
4765N/A {
4765N/A indexCount += 3;
4765N/A }
4765N/A else if (lowerName.equals("dn2uri"))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A else if (lowerName.startsWith("vlv."))
4765N/A {
4765N/A if(lowerName.length() < 5)
4765N/A {
4765N/A Message msg = ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName);
4765N/A throw new JebException(msg);
4765N/A }
4765N/A indexCount++;
4765N/A } else if(lowerName.equals("id2subtree") ||
4765N/A lowerName.equals("id2children"))
4765N/A {
4765N/A Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
4765N/A throw new JebException(msg);
4765N/A }
4765N/A else
4765N/A {
4765N/A String[] attrIndexParts = lowerName.split("\\.");
4765N/A if((attrIndexParts.length <= 0) || (attrIndexParts.length > 3))
4765N/A {
4765N/A Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
4765N/A throw new JebException(msg);
4765N/A }
4765N/A AttributeType attrType =
4765N/A DirectoryServer.getAttributeType(attrIndexParts[0]);
4765N/A if (attrType == null)
4765N/A {
4765N/A Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
4765N/A throw new JebException(msg);
4765N/A }
4765N/A if(attrIndexParts.length != 1)
4765N/A {
4765N/A if(attrIndexParts.length == 2)
4765N/A {
4765N/A if(attrIndexParts[1].equals("presence"))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A else if(attrIndexParts[1].equals("equality"))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A else if(attrIndexParts[1].equals("substring"))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A else if(attrIndexParts[1].equals("ordering"))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A else if(attrIndexParts[1].equals("approximate"))
4765N/A {
4765N/A indexCount++;
4765N/A } else {
4765N/A Message msg =
4765N/A ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
4765N/A throw new JebException(msg);
4765N/A }
4765N/A }
4765N/A else
4765N/A {
4765N/A boolean found = false;
4765N/A String s = attrIndexParts[1] + "." + attrIndexParts[2];
4765N/A for (String idx : cfg.listLocalDBIndexes())
4765N/A {
4765N/A LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
4765N/A if (indexCfg.getIndexType().
4765N/A contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
4765N/A {
4765N/A Set<String> extensibleRules =
4765N/A indexCfg.getIndexExtensibleMatchingRule();
4765N/A for(String exRule : extensibleRules)
4765N/A {
4765N/A if(exRule.equalsIgnoreCase(s))
4765N/A {
4765N/A found = true;
4765N/A break;
4765N/A }
4765N/A }
4765N/A }
4765N/A if(found)
4765N/A {
4765N/A break;
4765N/A }
4765N/A }
4765N/A if(!found) {
4765N/A Message msg =
4765N/A ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
4765N/A throw new JebException(msg);
4765N/A }
4765N/A indexCount++;
4765N/A }
4765N/A }
4765N/A else
4765N/A {
4765N/A for (String idx : cfg.listLocalDBIndexes())
4765N/A {
4765N/A if(!idx.equalsIgnoreCase(index))
4765N/A {
4765N/A continue;
4765N/A }
4765N/A LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
4765N/A if(indexCfg.getIndexType().
4765N/A contains(LocalDBIndexCfgDefn.IndexType.EQUALITY))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A if(indexCfg.getIndexType().
4765N/A contains(LocalDBIndexCfgDefn.IndexType.ORDERING))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A if(indexCfg.getIndexType().
4765N/A contains(LocalDBIndexCfgDefn.IndexType.PRESENCE))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A if(indexCfg.getIndexType().
4765N/A contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A if(indexCfg.getIndexType().
4765N/A contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A if (indexCfg.getIndexType().
4765N/A contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
4765N/A {
4765N/A Set<String> extensibleRules =
4765N/A indexCfg.getIndexExtensibleMatchingRule();
4765N/A boolean shared = false;
4765N/A for(String exRule : extensibleRules)
4765N/A {
4765N/A if(exRule.endsWith(".sub"))
4765N/A {
4765N/A indexCount++;
4765N/A }
4765N/A else
4765N/A {
4765N/A if(!shared)
4765N/A {
4765N/A shared=true;
4765N/A indexCount++;
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A return indexCount;
4765N/A }
4765N/A
4765N/A private void clearRebuildListIndexes() throws DatabaseException
4765N/A {
4765N/A List<String> rebuildList = rebuildConfig.getRebuildList();
4765N/A if(!rebuildList.isEmpty())
4765N/A {
4765N/A for (String index : rebuildList)
4765N/A {
4765N/A String lowerName = index.toLowerCase();
4765N/A if (lowerName.equals("dn2id"))
4765N/A {
4765N/A clearDN2IDIndexes(ec);
4765N/A }
4765N/A else if (lowerName.equals("dn2uri"))
4765N/A {
4765N/A clearDN2URI(ec);
4765N/A }
4765N/A else if (lowerName.startsWith("vlv."))
4765N/A {
4765N/A clearVLVIndex(lowerName.substring(4), ec);
4765N/A }
4765N/A else
4765N/A {
4765N/A String[] attrIndexParts = lowerName.split("\\.");
4765N/A AttributeType attrType =
4765N/A DirectoryServer.getAttributeType(attrIndexParts[0]);
4765N/A AttributeIndex attrIndex = ec.getAttributeIndex(attrType);
4765N/A
4765N/A if(attrIndexParts.length != 1)
4765N/A {
4765N/A Index partialAttrIndex;
4765N/A if(attrIndexParts[1].equals("presence"))
4765N/A {
4765N/A partialAttrIndex = attrIndex.getPresenceIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.PRESENCE);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A else if(attrIndexParts[1].equals("equality"))
4765N/A {
4765N/A partialAttrIndex = attrIndex.getEqualityIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.EQUALITY);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A else if(attrIndexParts[1].equals("substring"))
4765N/A {
4765N/A partialAttrIndex = attrIndex.getSubstringIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.SUBSTRING);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A else if(attrIndexParts[1].equals("ordering"))
4765N/A {
4765N/A partialAttrIndex = attrIndex.getOrderingIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.ORDERING);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A else if(attrIndexParts[1].equals("approximate"))
4765N/A {
4765N/A partialAttrIndex = attrIndex.getApproximateIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.APPROXIMATE);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A else
4765N/A {
4765N/A String dbPart = "shared";
4765N/A if(attrIndexParts[2].startsWith("sub"))
4765N/A {
4765N/A dbPart = "substring";
4765N/A }
4765N/A StringBuilder nameBldr = new StringBuilder();
4765N/A nameBldr.append(ec.getDatabasePrefix());
4765N/A nameBldr.append("_");
4765N/A nameBldr.append(attrIndexParts[0]);
4765N/A nameBldr.append(".");
4765N/A nameBldr.append(attrIndexParts[1]);
4765N/A nameBldr.append(".");
4765N/A nameBldr.append(dbPart);
4765N/A String indexName = nameBldr.toString();
4765N/A Map<String,Collection<Index>> extensibleMap =
4765N/A attrIndex.getExtensibleIndexes();
4765N/A if(!extensibleMap.isEmpty()) {
4765N/A Collection<Index> subIndexes =
4765N/A attrIndex.getExtensibleIndexes().get(
4765N/A EXTENSIBLE_INDEXER_ID_SUBSTRING);
4765N/A if(subIndexes != null) {
4765N/A for(Index subIndex : subIndexes) {
4765N/A String name = subIndex.getName();
4765N/A if(name.equalsIgnoreCase(indexName))
4765N/A {
4765N/A ec.clearDatabase(subIndex);
4765N/A Collection<Index> substring = new ArrayList<Index>();
4765N/A substring.add(subIndex);
4765N/A extensibleIndexMap.put(new IndexKey(attrType,
4765N/A ImportIndexType.EX_SUBSTRING),substring);
4765N/A break;
4765N/A }
4765N/A }
4765N/A Collection<Index> sharedIndexes = attrIndex.
4765N/A getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
4765N/A if(sharedIndexes !=null) {
4765N/A for(Index sharedIndex : sharedIndexes) {
4765N/A String name = sharedIndex.getName();
4765N/A if(name.equalsIgnoreCase(indexName))
4765N/A {
4765N/A ec.clearDatabase(sharedIndex);
4765N/A Collection<Index> shared = new ArrayList<Index>();
4765N/A shared.add(sharedIndex);
4765N/A extensibleIndexMap.put(new IndexKey(attrType,
4765N/A ImportIndexType.EX_SHARED), shared);
4765N/A break;
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A else
4765N/A {
4765N/A clearAttributeIndexes(attrIndex, attrType, ec);
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A
4765N/A
4765N/A private void clearAllIndexes() throws DatabaseException
4765N/A {
4765N/A for(Map.Entry<AttributeType, AttributeIndex> mapEntry :
4765N/A suffix.getAttrIndexMap().entrySet()) {
4765N/A AttributeType attributeType = mapEntry.getKey();
4765N/A AttributeIndex attributeIndex = mapEntry.getValue();
4765N/A clearAttributeIndexes(attributeIndex, attributeType, ec);
4765N/A }
4765N/A for(VLVIndex vlvIndex : suffix.getEntryContainer().getVLVIndexes()) {
4765N/A ec.clearDatabase(vlvIndex);
4765N/A }
4765N/A clearDN2IDIndexes(ec);
4765N/A if(ec.getDN2URI() != null)
4765N/A {
4765N/A clearDN2URI(ec);
4765N/A }
4765N/A }
4765N/A
4765N/A private void clearVLVIndex(String name, EntryContainer ec)
4765N/A throws DatabaseException
4765N/A {
4765N/A VLVIndex vlvIndex = ec.getVLVIndex(name);
4765N/A ec.clearDatabase(vlvIndex);
4765N/A }
4765N/A
4765N/A private void clearDN2URI(EntryContainer ec) throws DatabaseException
4765N/A {
4765N/A ec.clearDatabase(ec.getDN2URI());
4765N/A dn2uri = ec.getDN2URI();
4765N/A }
4765N/A
4765N/A private void clearDN2IDIndexes(EntryContainer ec) throws DatabaseException
4765N/A {
4765N/A ec.clearDatabase(ec.getDN2ID());
4765N/A ec.clearDatabase(ec.getID2Children());
4765N/A ec.clearDatabase(ec.getID2Subtree());
4765N/A dn2id = ec.getDN2ID();
4765N/A }
4765N/A
4765N/A private void clearAttributeIndexes(AttributeIndex attrIndex,
4765N/A AttributeType attrType, EntryContainer ec)
4765N/A throws DatabaseException
4765N/A {
4765N/A Index partialAttrIndex;
4765N/A if(attrIndex.getSubstringIndex() != null)
4765N/A {
4765N/A partialAttrIndex = attrIndex.getSubstringIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.SUBSTRING);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A if(attrIndex.getOrderingIndex() != null)
4765N/A {
4765N/A partialAttrIndex = attrIndex.getOrderingIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.ORDERING);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A if(attrIndex.getEqualityIndex() != null)
4765N/A {
4765N/A partialAttrIndex = attrIndex.getEqualityIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.EQUALITY);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A if(attrIndex.getPresenceIndex() != null)
4765N/A {
4765N/A partialAttrIndex = attrIndex.getPresenceIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.PRESENCE);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A
4765N/A }
4765N/A if(attrIndex.getApproximateIndex() != null)
4765N/A {
4765N/A partialAttrIndex = attrIndex.getApproximateIndex();
4765N/A ec.clearDatabase(partialAttrIndex);
4765N/A IndexKey indexKey =
4765N/A new IndexKey(attrType, ImportIndexType.APPROXIMATE);
4765N/A indexMap.put(indexKey, partialAttrIndex);
4765N/A }
4765N/A Map<String,Collection<Index>> extensibleMap =
4765N/A attrIndex.getExtensibleIndexes();
4765N/A if(!extensibleMap.isEmpty()) {
4765N/A Collection<Index> subIndexes =
4765N/A attrIndex.getExtensibleIndexes().get(
4765N/A EXTENSIBLE_INDEXER_ID_SUBSTRING);
4765N/A if(subIndexes != null) {
4765N/A for(Index subIndex : subIndexes) {
4765N/A ec.clearDatabase(subIndex);
4765N/A }
4765N/A extensibleIndexMap.put(new IndexKey(attrType,
4765N/A ImportIndexType.EX_SUBSTRING), subIndexes);
4765N/A }
4765N/A Collection<Index> sharedIndexes =
4765N/A attrIndex.getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
4765N/A if(sharedIndexes !=null) {
4765N/A for(Index sharedIndex : sharedIndexes) {
4765N/A ec.clearDatabase(sharedIndex);
4765N/A }
4765N/A extensibleIndexMap.put(new IndexKey(attrType,
4765N/A ImportIndexType.EX_SHARED), sharedIndexes);
4765N/A }
4765N/A }
4765N/A }
4765N/A
4765N/A
4765N/A private
4765N/A void processEntry(Entry entry, EntryID entryID) throws DatabaseException,
4765N/A ConfigException, DirectoryException, JebException
4765N/A {
4765N/A if(dn2id != null)
4765N/A {
4765N/A processDN2ID(suffix, entry.getDN(), entryID);
4765N/A }
4765N/A if(dn2uri != null)
4765N/A {
4765N/A processDN2URI(suffix, null, entry);
4765N/A }
4765N/A processIndexes(entry, entryID);
4765N/A processExtensibleIndexes(entry, entryID);
4765N/A processVLVIndexes(entry, entryID);
4765N/A }
4765N/A
4765N/A private void processVLVIndexes(Entry entry, EntryID entryID)
4765N/A throws DatabaseException, JebException, DirectoryException
4765N/A {
4765N/A for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
4765N/A Transaction transaction = null;
4765N/A vlvIdx.addEntry(transaction, entryID, entry);
4765N/A }
4765N/A }
4765N/A
4765N/A
4765N/A private void processExtensibleIndexes(Entry entry, EntryID entryID) throws
4765N/A DatabaseException, DirectoryException, JebException, ConfigException
4765N/A {
4765N/A for(Map.Entry<IndexKey, Collection<Index>> mapEntry :
4765N/A this.extensibleIndexMap.entrySet()) {
4765N/A IndexKey key = mapEntry.getKey();
4765N/A AttributeType attrType = key.getType();
4765N/A if(entry.hasAttribute(attrType)) {
4765N/A Collection<Index> indexes = mapEntry.getValue();
4765N/A for(Index index : indexes) {
4765N/A processAttribute(index, entry, entryID, key);
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A
4765N/A private void
4765N/A processIndexes(Entry entry, EntryID entryID) throws
4765N/A DatabaseException, DirectoryException, JebException, ConfigException
4765N/A {
4765N/A
4765N/A for(Map.Entry<IndexKey, Index> mapEntry :
4765N/A indexMap.entrySet()) {
4765N/A IndexKey key = mapEntry.getKey();
4765N/A AttributeType attrType = key.getType();
4765N/A if(entry.hasAttribute(attrType)) {
4765N/A ImportIndexType indexType = key.getIndexType();
4765N/A Index index = mapEntry.getValue();
4765N/A if(indexType == ImportIndexType.SUBSTRING)
4765N/A {
4765N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attrType, ImportIndexType.SUBSTRING));
4765N/A }
4765N/A else
4765N/A {
4765N/A processAttribute(index, entry, entryID,
4765N/A new IndexKey(attrType, indexType));
4765N/A }
4765N/A }
4765N/A }
4765N/A }
4765N/A
4765N/A /**
4765N/A * Return the number of entries processed by the rebuild manager.
4765N/A *
4765N/A * @return The number of entries processed.
4765N/A */
4765N/A public long getEntriesProcess()
4765N/A {
4765N/A return this.entriesProcessed.get();
4765N/A }
4765N/A
4765N/A /**
4765N/A * Return the total number of entries to process by the rebuild manager.
4765N/A *
4765N/A * @return The total number for entries to process.
4765N/A */
4765N/A public long getTotEntries()
4765N/A {
4765N/A return this.totalEntries;
4765N/A }
4765N/A }
4765N/A
4765N/A /**
4765N/A * This class reports progress of the rebuild job at fixed intervals.
4765N/A */
4765N/A class RBFirstPhaseProgressTask extends TimerTask
4765N/A {
4765N/A /**
4765N/A * The number of records that had been processed at the time of the
4765N/A * previous progress report.
4765N/A */
4765N/A private long previousProcessed = 0;
4765N/A
4765N/A /**
4765N/A * The time in milliseconds of the previous progress report.
4765N/A */
4765N/A private long previousTime;
4765N/A
4765N/A /**
4765N/A * The environment statistics at the time of the previous report.
4765N/A */
4765N/A private EnvironmentStats prevEnvStats;
4765N/A
4765N/A /**
4765N/A * Create a new verify progress task.
4765N/A * @throws DatabaseException An error occurred while accessing the JE
4765N/A * database.
4765N/A */
4765N/A public RBFirstPhaseProgressTask() throws DatabaseException
4765N/A {
4765N/A previousTime = System.currentTimeMillis();
4765N/A prevEnvStats =
4765N/A rootContainer.getEnvironmentStats(new StatsConfig());
4765N/A }
4765N/A
4765N/A /**
4765N/A * The action to be performed by this timer task.
4765N/A */
4765N/A public void run()
4765N/A {
4765N/A long latestTime = System.currentTimeMillis();
4765N/A long deltaTime = latestTime - previousTime;
4765N/A
4765N/A if (deltaTime == 0)
4765N/A {
4765N/A return;
4765N/A }
4765N/A long currentRBProcessed = rebuildManager.getEntriesProcess();
4765N/A long deltaCount = (currentRBProcessed - previousProcessed);
4765N/A float rate = 1000f*deltaCount / deltaTime;
4765N/A float completed = 0;
4765N/A if(rebuildManager.getTotEntries() > 0)
4765N/A {
4765N/A completed = 100f*currentRBProcessed / rebuildManager.getTotEntries();
4765N/A }
4765N/A Message message = NOTE_JEB_REBUILD_PROGRESS_REPORT.get(
4765N/A completed, currentRBProcessed, rebuildManager.getTotEntries(), rate);
4765N/A logError(message);
4765N/A try
4765N/A {
4765N/A Runtime runtime = Runtime.getRuntime();
4765N/A long freeMemory = runtime.freeMemory() / MB;
4765N/A EnvironmentStats envStats =
4765N/A rootContainer.getEnvironmentStats(new StatsConfig());
4765N/A long nCacheMiss =
4765N/A envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();
4765N/A
4765N/A float cacheMissRate = 0;
4765N/A if (deltaCount > 0)
4765N/A {
4765N/A cacheMissRate = nCacheMiss/(float)deltaCount;
4765N/A }
4765N/A message = NOTE_JEB_REBUILD_CACHE_AND_MEMORY_REPORT.get(
4765N/A freeMemory, cacheMissRate);
4765N/A logError(message);
4765N/A prevEnvStats = envStats;
4765N/A }
4765N/A catch (DatabaseException e)
4765N/A {
4765N/A
4765N/A }
4765N/A previousProcessed = currentRBProcessed;
4765N/A previousTime = latestTime;
4765N/A }
4765N/A }
4765N/A
4765N/A
4591N/A /**
3339N/A * This class reports progress of the import job at fixed intervals.
3339N/A */
4591N/A private final class FirstPhaseProgressTask extends TimerTask
3339N/A {
3339N/A /**
3339N/A * The number of entries that had been read at the time of the
3339N/A * previous progress report.
3339N/A */
3339N/A private long previousCount = 0;
3339N/A
3339N/A /**
3339N/A * The time in milliseconds of the previous progress report.
3339N/A */
3339N/A private long previousTime;
3339N/A
3339N/A /**
3339N/A * The environment statistics at the time of the previous report.
3339N/A */
4649N/A private EnvironmentStats previousStats;
3339N/A
3339N/A
4591N/A // Determines if eviction has been detected.
3339N/A private boolean evicting = false;
3339N/A
4591N/A // Entry count when eviction was detected.
3339N/A private long evictionEntryCount = 0;
3339N/A
3339N/A
4649N/A /**
3339N/A * Create a new import progress task.
3339N/A */
4591N/A public FirstPhaseProgressTask()
3339N/A {
3339N/A previousTime = System.currentTimeMillis();
4591N/A try
4591N/A {
4649N/A previousStats =
4591N/A rootContainer.getEnvironmentStats(new StatsConfig());
4591N/A }
4591N/A catch (DatabaseException e)
4591N/A {
4591N/A throw new RuntimeException(e);
4591N/A }
3339N/A }
3339N/A
3339N/A
3339N/A
4649N/A /**
4649N/A * The action to be performed by this timer task.
4649N/A */
4649N/A @Override
4649N/A public void run()
4591N/A {
4649N/A long latestCount = reader.getEntriesRead() + 0;
4649N/A long deltaCount = (latestCount - previousCount);
4649N/A long latestTime = System.currentTimeMillis();
4649N/A long deltaTime = latestTime - previousTime;
4649N/A Message message;
4649N/A if (deltaTime == 0)
4649N/A {
4649N/A return;
4649N/A }
4649N/A long entriesRead = reader.getEntriesRead();
4649N/A long entriesIgnored = reader.getEntriesIgnored();
4649N/A long entriesRejected = reader.getEntriesRejected();
4649N/A float rate = 1000f * deltaCount / deltaTime;
4649N/A message = NOTE_JEB_IMPORT_PROGRESS_REPORT.get(entriesRead,
4649N/A entriesIgnored, entriesRejected, 0, rate);
4649N/A logError(message);
4649N/A try
4649N/A {
4649N/A Runtime runTime = Runtime.getRuntime();
4649N/A long freeMemory = runTime.freeMemory() / MB;
4649N/A EnvironmentStats environmentStats =
4649N/A rootContainer.getEnvironmentStats(new StatsConfig());
4649N/A long nCacheMiss = environmentStats.getNCacheMiss() -
4649N/A previousStats.getNCacheMiss();
3339N/A
4649N/A float cacheMissRate = 0;
4649N/A if (deltaCount > 0)
4649N/A {
4649N/A cacheMissRate = nCacheMiss / (float) deltaCount;
4649N/A }
3339N/A message =
4649N/A NOTE_JEB_IMPORT_CACHE_AND_MEMORY_REPORT.get(freeMemory,
4649N/A cacheMissRate);
3339N/A logError(message);
4649N/A long evictPasses = environmentStats.getNEvictPasses();
4649N/A long evictNodes = environmentStats.getNNodesExplicitlyEvicted();
4649N/A long evictBinsStrip = environmentStats.getNBINsStripped();
4649N/A long cleanerRuns = environmentStats.getNCleanerRuns();
4649N/A long cleanerDeletions = environmentStats.getNCleanerDeletions();
4649N/A long cleanerEntriesRead =
4649N/A environmentStats.getNCleanerEntriesRead();
4649N/A long cleanerINCleaned = environmentStats.getNINsCleaned();
4649N/A long checkPoints = environmentStats.getNCheckpoints();
4649N/A if (evictPasses != 0)
4649N/A {
4649N/A if (!evicting)
4649N/A {
4649N/A evicting = true;
4649N/A evictionEntryCount = reader.getEntriesRead();
4649N/A message =
4649N/A NOTE_JEB_IMPORT_LDIF_EVICTION_DETECTED
4649N/A .get(evictionEntryCount);
4649N/A logError(message);
4649N/A }
4649N/A message =
4649N/A NOTE_JEB_IMPORT_LDIF_EVICTION_DETECTED_STATS.get(
4649N/A evictPasses, evictNodes, evictBinsStrip);
4649N/A logError(message);
4649N/A }
4649N/A if (cleanerRuns != 0)
4649N/A {
4649N/A message =
4649N/A NOTE_JEB_IMPORT_LDIF_CLEANER_STATS.get(cleanerRuns,
4649N/A cleanerDeletions, cleanerEntriesRead,
4649N/A cleanerINCleaned);
4649N/A logError(message);
4649N/A }
4649N/A if (checkPoints > 1)
4649N/A {
4649N/A message =
4649N/A NOTE_JEB_IMPORT_LDIF_BUFFER_CHECKPOINTS.get(checkPoints);
4649N/A logError(message);
4649N/A }
4649N/A previousStats = environmentStats;
3339N/A }
4649N/A catch (DatabaseException e)
4649N/A {
4649N/A // Unlikely to happen and not critical.
4649N/A }
4649N/A previousCount = latestCount;
4649N/A previousTime = latestTime;
4591N/A }
3339N/A }
4591N/A
4591N/A
4591N/A
4591N/A /**
4591N/A * This class reports progress of the import job at fixed intervals.
4591N/A */
4765N/A class SecondPhaseProgressTask extends TimerTask
4591N/A {
4591N/A /**
4591N/A * The number of entries that had been read at the time of the
4591N/A * previous progress report.
4591N/A */
4591N/A private long previousCount = 0;
4591N/A
4591N/A /**
4591N/A * The time in milliseconds of the previous progress report.
4591N/A */
4591N/A private long previousTime;
4591N/A
4591N/A /**
4591N/A * The environment statistics at the time of the previous report.
4591N/A */
4649N/A private EnvironmentStats previousStats;
4591N/A
4591N/A // Determines if eviction has been detected.
4591N/A private boolean evicting = false;
4591N/A
4643N/A private final List<IndexManager> indexMgrList;
4765N/A private long latestCount;
4591N/A
4649N/A /**
4591N/A * Create a new import progress task.
4643N/A * @param indexMgrList List of index managers.
4765N/A * @param latestCount The latest count of entries processed in phase one.
4591N/A */
4765N/A public SecondPhaseProgressTask (List<IndexManager> indexMgrList,
4765N/A long latestCount)
4591N/A {
4591N/A previousTime = System.currentTimeMillis();
4643N/A this.indexMgrList = indexMgrList;
4765N/A this.latestCount = latestCount;
4591N/A try
4591N/A {
4649N/A previousStats =
4591N/A rootContainer.getEnvironmentStats(new StatsConfig());
4591N/A }
4591N/A catch (DatabaseException e)
4591N/A {
4591N/A throw new RuntimeException(e);
4591N/A }
4591N/A }
4591N/A
4591N/A
4591N/A /**
4591N/A * The action to be performed by this timer task.
4591N/A */
4591N/A @Override
4591N/A public void run()
4591N/A {
4591N/A long deltaCount = (latestCount - previousCount);
4591N/A long latestTime = System.currentTimeMillis();
4591N/A long deltaTime = latestTime - previousTime;
4591N/A Message message;
4591N/A if (deltaTime == 0)
4591N/A {
4591N/A return;
4591N/A }
4591N/A try
4591N/A {
4649N/A Runtime runTime = Runtime.getRuntime();
4649N/A long freeMemory = runTime.freeMemory() / MB;
4649N/A EnvironmentStats environmentStats =
4591N/A rootContainer.getEnvironmentStats(new StatsConfig());
4649N/A long nCacheMiss = environmentStats.getNCacheMiss() -
4649N/A previousStats.getNCacheMiss();
4591N/A
4591N/A float cacheMissRate = 0;
4591N/A if (deltaCount > 0)
4591N/A {
4591N/A cacheMissRate = nCacheMiss / (float) deltaCount;
4591N/A }
4591N/A message =
4591N/A NOTE_JEB_IMPORT_CACHE_AND_MEMORY_REPORT.get(freeMemory,
4591N/A cacheMissRate);
4591N/A logError(message);
4649N/A long evictPasses = environmentStats.getNEvictPasses();
4649N/A long evictNodes = environmentStats.getNNodesExplicitlyEvicted();
4649N/A long evictBinsStrip = environmentStats.getNBINsStripped();
4649N/A long cleanerRuns = environmentStats.getNCleanerRuns();
4649N/A long cleanerDeletions = environmentStats.getNCleanerDeletions();
4649N/A long cleanerEntriesRead = environmentStats.getNCleanerEntriesRead();
4649N/A long cleanerINCleaned = environmentStats.getNINsCleaned();
4649N/A long checkPoints = environmentStats.getNCheckpoints();
4591N/A if (evictPasses != 0)
4591N/A {
4591N/A if (!evicting)
4591N/A {
4591N/A evicting = true;
4591N/A }
4591N/A message =
4591N/A NOTE_JEB_IMPORT_LDIF_EVICTION_DETECTED_STATS.get(
4591N/A evictPasses, evictNodes, evictBinsStrip);
4591N/A logError(message);
4591N/A }
4591N/A if (cleanerRuns != 0)
4591N/A {
4591N/A message =
4591N/A NOTE_JEB_IMPORT_LDIF_CLEANER_STATS.get(cleanerRuns,
4591N/A cleanerDeletions, cleanerEntriesRead,
4591N/A cleanerINCleaned);
4591N/A logError(message);
4591N/A }
4591N/A if (checkPoints > 1)
4591N/A {
4591N/A message =
4591N/A NOTE_JEB_IMPORT_LDIF_BUFFER_CHECKPOINTS.get(checkPoints);
4591N/A logError(message);
4591N/A }
4649N/A previousStats = environmentStats;
4591N/A }
4591N/A catch (DatabaseException e)
4591N/A {
4591N/A // Unlikely to happen and not critical.
4591N/A }
4591N/A previousCount = latestCount;
4591N/A previousTime = latestTime;
4591N/A
4643N/A for(IndexManager indexMgr : indexMgrList)
4591N/A {
4591N/A indexMgr.printStats(deltaTime);
4591N/A }
4591N/A }
4591N/A }
4643N/A
4643N/A
4643N/A /**
4643N/A * A class to hold information about the entry determined by the LDIF reader.
4643N/A *
4643N/A */
4643N/A public class EntryInformation
4643N/A {
4643N/A private EntryID entryID;
4643N/A private Suffix suffix;
4643N/A
4643N/A
4643N/A /**
4643N/A * Return the suffix associated with the entry.
4643N/A *
4643N/A * @return Entry's suffix instance;
4643N/A */
4643N/A public Suffix getSuffix()
4643N/A {
4643N/A return suffix;
4643N/A }
4643N/A
4643N/A /**
4643N/A * Set the suffix instance associated with the entry.
4643N/A *
4643N/A * @param suffix The suffix associated with the entry.
4643N/A */
4643N/A public void setSuffix(Suffix suffix)
4643N/A {
4643N/A this.suffix = suffix;
4643N/A }
4643N/A
4643N/A /**
4643N/A * Set the entry's ID.
4643N/A *
4643N/A * @param entryID The entry ID to set the entry ID to.
4643N/A */
4643N/A public void setEntryID(EntryID entryID)
4643N/A {
4643N/A this.entryID = entryID;
4643N/A }
4643N/A
4643N/A /**
4643N/A * Return the entry ID associated with the entry.
4643N/A *
4643N/A * @return The entry ID associated with the entry.
4643N/A */
4643N/A public EntryID getEntryID()
4643N/A {
4643N/A return entryID;
4643N/A }
4643N/A }
4643N/A
4643N/A /**
4643N/A * This class defines the individual index type available.
4643N/A *
4643N/A */
4765N/A public enum ImportIndexType {
4643N/A /**
4643N/A * The DN index type.
4643N/A **/
4643N/A DN,
4643N/A
4643N/A /**
4643N/A * The equality index type.
4643N/A **/
4643N/A EQUALITY,
4643N/A
4643N/A /**
4643N/A * The presence index type.
4643N/A **/
4643N/A PRESENCE,
4643N/A
4643N/A /**
4649N/A * The sub-string index type.
4643N/A **/
4643N/A SUBSTRING,
4643N/A
4643N/A /**
4643N/A * The ordering index type.
4643N/A **/
4643N/A ORDERING,
4643N/A
4643N/A /**
4643N/A * The approximate index type.
4643N/A **/
4643N/A APPROXIMATE,
4643N/A
4643N/A /**
4649N/A * The extensible sub-string index type.
4643N/A **/
4643N/A EX_SUBSTRING,
4643N/A
4643N/A /**
4643N/A * The extensible shared index type.
4643N/A **/
4765N/A EX_SHARED,
4765N/A
4765N/A /**
4765N/A * The vlv index type.
4765N/A */
4765N/A VLV
4643N/A }
4643N/A
4643N/A /**
4649N/A * This class is used as an index key for hash maps that need to
4649N/A * process multiple suffix index elements into a single queue and/or maps
4649N/A * based on both attribute type and index type
4649N/A * (ie., cn.equality, sn.equality,...).
4643N/A *
4649N/A * It tries to perform some optimization if the index is a sub-string index.
4643N/A */
4643N/A public class IndexKey {
4643N/A
4643N/A private final AttributeType type;
4765N/A private final ImportIndexType indexType;
4765N/A
4643N/A
4643N/A /**
4643N/A * Create index key instance using the specified attribute type, index type.
4643N/A *
4643N/A * @param type The attribute type.
4643N/A * @param indexType The index type.
4643N/A */
4765N/A IndexKey(AttributeType type, ImportIndexType indexType)
4643N/A {
4643N/A this.type = type;
4643N/A this.indexType = indexType;
4643N/A }
4643N/A
4765N/A /**
4765N/A * An equals method that uses both the attribute type and the index type.
4765N/A *
4765N/A * @param obj the object to compare.
4765N/A * @return <CODE>true</CODE> if the objects are equal.
4765N/A */
4765N/A public boolean equals(Object obj)
4765N/A {
4765N/A boolean returnCode = false;
4765N/A if (obj instanceof IndexKey) {
4765N/A IndexKey oKey = (IndexKey) obj;
4765N/A if(type.equals(oKey.getType()) &&
4765N/A indexType.equals(oKey.getIndexType()))
4765N/A {
4765N/A returnCode = true;
4765N/A }
4643N/A }
4765N/A return returnCode;
4765N/A }
4643N/A
4643N/A /**
4649N/A * A hash code method that adds the hash codes of the attribute type and
4643N/A * index type and returns that value.
4643N/A *
4643N/A * @return The combined hash values.
4643N/A */
4643N/A public int hashCode()
4643N/A {
4643N/A return type.hashCode() + indexType.hashCode();
4643N/A }
4643N/A
4643N/A /**
4643N/A * Return the attribute type.
4643N/A *
4643N/A * @return The attribute type.
4643N/A */
4643N/A public AttributeType getType()
4643N/A {
4643N/A return type;
4643N/A }
4643N/A
4643N/A /**
4643N/A * Return the index type.
4643N/A * @return The index type.
4643N/A */
4765N/A public ImportIndexType getIndexType()
4643N/A {
4643N/A return indexType;
4643N/A }
4643N/A
4643N/A /**
4643N/A * Return the index key name, which is the attribute type primary name,
4643N/A * a period, and the index type name. Used for building file names and
4643N/A * output.
4643N/A *
4643N/A * @return The index key name.
4643N/A */
4643N/A public String getName()
4643N/A {
4643N/A return type.getPrimaryName() + "." +
4765N/A StaticUtils.toLowerCase(indexType.name());
4643N/A }
4643N/A }
3339N/A}