Importer.java revision f0a048d41a13eca4cba405da9403c2469ca3d1ea
2362N/A * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
0N/A * file and include the License file at legal-notices/CDDLv1_0.txt.
0N/A private int migratedCount;
0N/A private int bufferSize;
0N/A private final int indexCount;
0N/A private int threadCount;
0N/A /** Whether DN validation should be performed. If true, then it is performed during phase one. */
0N/A private final boolean validateDNs;
0N/A private long dnCacheSize;
0N/A private long availableMemory;
0N/A private long dbCacheSize;
0N/A private final Map<IndexKey, BlockingQueue<IndexOutputBuffer>> indexKeyQueueMap = new ConcurrentHashMap<>();
2581N/A private final ConcurrentHashMap<Integer, Index> indexIDToIndexMap = new ConcurrentHashMap<>();
2581N/A private final ConcurrentHashMap<Integer, EntryContainer> indexIDToECMap = new ConcurrentHashMap<>();
0N/A private final boolean clearedBackend;
0N/A private volatile boolean isCanceled;
0N/A private int phaseOneBufferCount;
0N/A ServerContext serverContext) throws InitializationException, StorageRuntimeException, ConfigException
0N/A this.clearedBackend = false;
2028N/A this.validateDNs = false;
0N/A Importer(RootContainer rootContainer, LDIFImportConfig importCfg, PluggableBackendCfg backendCfg,
0N/A ServerContext serverContext) throws InitializationException, ConfigException, StorageRuntimeException
647N/A if (validateDNs)
2581N/A private File prepareTempDir(PluggableBackendCfg backendCfg, String tmpDirectory) throws InitializationException
2581N/A private static int getTotalIndexCount(PluggableBackendCfg backendCfg) throws ConfigException
0N/A else if (!clearedBackend)
0N/A if (validateDNs)
0N/A if (!clearedBackend)
0N/A threadCount--;
0N/A private void calculateAvailableMemory()
0N/A final long totalAvailableMemory;
0N/A private void initializeIndexBuffers()
0N/A throws ConfigException
2028N/A private EntryContainer createEntryContainer(WriteableTransaction txn, DN baseDN) throws ConfigException
2028N/A catch (DirectoryException e)
0N/A public void rebuildIndexes() throws ConfigException, InitializationException, StorageRuntimeException,
0N/A logger.info(NOTE_IMPORT_STARTING, DirectoryServer.getVersionString(), BUILD_ID, REVISION_NUMBER);
0N/A if (validateDNs)
0N/A if (isCanceled)
0N/A if (isCanceled)
0N/A if (validateDNs)
0N/A private void switchEntryContainers(WriteableTransaction txn) throws StorageRuntimeException, InitializationException
1139N/A private void setIndexesTrusted(WriteableTransaction txn, boolean trusted) throws StorageRuntimeException
0N/A throw new StorageRuntimeException(NOTE_IMPORT_LDIF_TRUSTED_FAILED.get(ex.getMessage()).toString());
0N/A private static void scheduleAtFixedRate(ScheduledThreadPoolExecutor timerService, Runnable task)
0N/A private static void shutdownAll(ExecutorService... executorServices) throws InterruptedException
0N/A int readAheadSize;
0N/A submitIndexDBWriteTasks(DNIndexMgrList, importer, dbService, permits, buffers, readAheadSize, futures);
0N/A submitIndexDBWriteTasks(indexMgrList, importer, dbService, permits, buffers, readAheadSize, futures);
0N/A ExecutorService dbService, Semaphore permits, int buffers, int readAheadSize, List<Future<Void>> futures)
0N/A futures.add(dbService.submit(new IndexDBWriteTask(importer, indexMgr, permits, buffers, readAheadSize)));
0N/A private static <T> void getAll(List<Future<T>> futures) throws InterruptedException, ExecutionException
0N/A private void stopScratchFileWriters()
0N/A && !isCanceled)
647N/A isCanceled = true;
0N/A && !isCanceled)
0N/A isCanceled = true;
0N/A return includeBranches;
0N/A isCanceled = true;
0N/A void processAllIndexes(Suffix suffix, Entry entry, EntryID entryID) throws StorageRuntimeException,
0N/A void processAttribute(MatchingRuleIndex index, Entry entry, EntryID entryID, IndexingOptions options,
0N/A isCanceled = true;
0N/A if (!clearedBackend)
0N/A void processIndexes(Suffix suffix, Entry entry, EntryID entryID) throws StorageRuntimeException,
0N/A void fillIndexKey(AttributeIndex attrIndex, Entry entry, AttributeType attrType, EntryID entryID)
0N/A throws DirectoryException
0N/A AttributeType attributeType, EntryID entryID, IndexingOptions options) throws InterruptedException
0N/A void processAttribute(MatchingRuleIndex index, Entry entry, EntryID entryID, IndexingOptions options,
0N/A return indexBuffer;
0N/A throws InterruptedException
0N/A throws StorageRuntimeException
0N/A private final int maxPermits;
0N/A private long lastBytesRead;
0N/A private int remainingBuffers;
0N/A private volatile int totalBatches;
0N/A private int nextBufferID;
0N/A private int ownedPermits;
0N/A public IndexDBWriteTask(org.opends.server.backends.pluggable.spi.Importer importer, IndexManager indexMgr,
0N/A logger.info(NOTE_IMPORT_LDIF_INDEX_STARTED, indexMgr.getBufferFileName(), remainingBuffers, totalBatches);
0N/A if (!isCanceled)
0N/A if (!isCanceled)
0N/A final int bytesReadPercent =
0N/A private void call0(org.opends.server.backends.pluggable.spi.Importer importer) throws Exception
0N/A if (isCanceled)
0N/A if (isCanceled)
0N/A if (b.hasMoreData())
0N/A b.fetchNextRecord();
0N/A logger.error(ERR_IMPORT_LDIF_INDEX_WRITE_DB_ERR, indexMgr.getBufferFileName(), e.getMessage());
private void addToDB(org.opends.server.backends.pluggable.spi.Importer importer, int indexID, ImportIDSet insertSet,
private void addDN2ID(org.opends.server.backends.pluggable.spi.Importer importer, int indexID, ImportIDSet idSet)
throws DirectoryException
private final class DNState
private long totalNbEntries;
return null;
throws StorageRuntimeException
if (bypassCacheForAppendMode())
return counter;
private boolean bypassCacheForAppendMode()
private EntryID get(org.opends.server.backends.pluggable.spi.Importer importer, TreeName dn2id, ByteSequence dn)
throws StorageRuntimeException
throws DirectoryException
private int bufferCount;
private boolean poisonSeen;
return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file), READER_WRITER_BUFFER_SIZE));
long bufferLen;
l.clear();
bufferCount++;
if (poisonSeen)
return null;
catch (IOException e)
logger.error(ERR_IMPORT_LDIF_INDEX_FILEWRITER_ERR, indexMgr.getBufferFile().getAbsolutePath(), e.getMessage());
isCanceled = true;
resetStreams();
resetStreams();
return bufferLen;
resetStreams();
if (b.isPoison())
poisonSeen = true;
resetStreams();
if (b.hasMoreData())
b.nextRecord();
return bufferLen;
private void resetStreams()
return getClass().getSimpleName() + "(" + indexMgr.getBufferFileName() + ": " + indexMgr.getBufferFile() + ")";
|| isCanceled)
isCanceled = true;
return null;
return null;
synchronized (synObj)
if (isDN2ID)
private final boolean isDN2ID;
private final int indexEntryLimit;
private int numberOfBuffers;
private long bufferFileSize;
private long totalDNs;
return bufferFile;
private long getBufferFileSize()
return bufferFileSize;
return bufferIndexFile;
private long getDNCount()
return totalDNs;
private boolean isDN2ID()
return isDN2ID;
return bufferFileName;
private int getIndexEntryLimit()
return indexEntryLimit;
private int getNumberOfBuffers()
return numberOfBuffers;
private long totalEntries;
super(storage);
case ALL:
case DEGRADED:
if (isCanceled)
catch (Exception e)
isCanceled = true;
if (isCanceled)
switch (mode)
case ALL:
case DEGRADED:
case USER_DEFINED:
private void rebuildAttributeIndexes(WriteableTransaction txn, AttributeIndex attrIndex, AttributeType attrType,
private void fillIndexMap(WriteableTransaction txn, AttributeType attrType, MatchingRuleIndex index,
private void setRebuildListIndexesTrusted(WriteableTransaction txn, boolean trusted) throws StorageRuntimeException
private void setTrusted(WriteableTransaction txn, final Collection<MatchingRuleIndex> indexes, boolean trusted)
return timer;
case ALL:
case DEGRADED:
indexCount++;
indexCount++;
indexCount++;
indexCount++;
boolean found = false;
found = true;
if (!found)
return indexCount;
private boolean findExtensibleMatchingRule(PluggableBackendCfg cfg, String indexExRuleName) throws ConfigException
result++;
return result;
boolean shared = false;
result++;
else if (!shared)
shared = true;
result++;
return result;
long getEntriesProcessed()
long getTotalEntries()
return this.totalEntries;
private long previousProcessed;
private long previousTime;
public void run()
logger.info(NOTE_REBUILD_PROGRESS_REPORT, completed, entriesProcessed, rebuildManager.getTotalEntries(), rate);
private long previousCount;
private long previousTime;
public FirstPhaseProgressTask()
public void run()
private long previousTime;
public SecondPhaseProgressTask()
public void run()
public class EntryInformation
return suffix;
return entryID;
public static class IndexKey
private final int entryLimit;
public int hashCode()
return attributeType;
return indexID;
private int getEntryLimit()
return entryLimit;
void close();
if ((methodName.startsWith("add") || methodName.startsWith("remove")) && methodName.endsWith("ChangeListener"))
return null;
return returnValue;
catch (Exception e)
throw new StorageRuntimeException(e);
private boolean insert(final ByteString key, final ByteStringBuilder dn) throws StorageRuntimeException
return existingDns;
final ByteStringBuilder builder = new ByteStringBuilder(dnList.length() + INT_SIZE + dntoAdd.length());
return builder;
return singleton;
catch (StorageRuntimeException e)
catch (Exception e)
throw new StorageRuntimeException(e);
catch (StorageRuntimeException e)
catch (Exception e)
throw new StorageRuntimeException(e);