VerifyJob.java revision 26e802d6fa4cb1dddd4764a931169f0be0effd61
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at legal-notices/CDDLv1_0.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information:
* Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*
*
* Copyright 2006-2010 Sun Microsystems, Inc.
* Portions Copyright 2011-2015 ForgeRock AS
*/
/** This class is used to run an index verification process on the backend. */
public class VerifyJob
{
/** The verify configuration. */
private final VerifyConfig verifyConfig;
/** The root container used for the verify job. */
private RootContainer rootContainer;
/** The number of milliseconds between job progress reports. */
private final long progressInterval = 10000;
/** The number of index keys processed. */
private long keyCount;
/** The number of errors found. */
private long errorCount;
/** The number of records that have exceeded the entry limit. */
private long entryLimitExceededCount;
/** The number of records that reference more than one entry. */
private long multiReferenceCount;
/** The total number of entry references. */
private long entryReferencesCount;
/** The maximum number of references per record. */
private long maxEntryPerValue;
/**
* This map is used to gather some statistics about values that have
* exceeded the entry limit.
*/
/** Indicates whether the DN database is to be verified. */
private boolean verifyDN2ID;
/** Indicates whether the children database is to be verified. */
private boolean verifyID2Children;
/** Indicates whether the subtree database is to be verified. */
private boolean verifyID2Subtree;
/** The entry database. */
/** The DN database. */
/** The children database. */
/** The subtree database. */
/** A list of the attribute indexes to be verified. */
/** A list of the VLV indexes to be verified. */
/**
* Construct a VerifyJob.
*
* @param verifyConfig The verify configuration.
*/
{
this.verifyConfig = verifyConfig;
}
/**
* Verify the backend.
*
* @param rootContainer The root container that holds the entries to verify.
* @return The error count.
* @throws StorageRuntimeException If an error occurs in the JE database.
* @throws DirectoryException If an error occurs while verifying the backend.
*/
public long verifyBackend(final RootContainer rootContainer) throws StorageRuntimeException, DirectoryException
{
try
{
{
{
}
});
}
catch (StorageRuntimeException e)
{
throw e;
}
catch (Exception e)
{
throw new StorageRuntimeException(e);
}
}
{
this.rootContainer = rootContainer;
try
{
boolean cleanMode = false;
{
verifyDN2ID = true;
{
verifyID2Children = true;
verifyID2Subtree = true;
}
}
else
{
if (!completeList.isEmpty())
{
list = completeList;
}
else
{
cleanMode = true;
}
{
{
verifyDN2ID = true;
}
{
{
verifyID2Children = true;
}
else
{
}
}
{
{
verifyID2Subtree = true;
}
else
{
}
}
{
{
}
{
}
}
else
{
{
}
{
}
}
}
}
// We will be updating these files independently of the indexes
// so we need direct access to them rather than going through
// the entry entryContainer methods.
// Make a note of the time we started.
// Start a timer for the progress report.
// Iterate through the index keys.
try
{
if (cleanMode)
{
}
else
{
// Make sure the vlv indexes are in correct order.
{
}
}
}
finally
{
}
float rate = 0;
if (totalTime > 0)
{
}
if (cleanMode)
{
if (multiReferenceCount > 0)
{
float averageEntryReferences = 0;
if (keyCount > 0)
{
}
if (logger.isDebugEnabled())
{
}
}
}
else
{
{
{
// Calculate the median value for entry limit exceeded.
long medianValue;
{
}
else
{
medianValue = values[x];
}
}
}
}
}
finally
{
}
return errorCount;
}
/**
* Iterate through the entries in id2entry to perform a check for
* index completeness. We check that the ID for the entry is indeed
* present in the indexes for the appropriate values.
*
* @throws StorageRuntimeException If an error occurs in the JE database.
*/
{
try
{
{
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
keyCount++;
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
}
if (keyCount != storedEntryCount)
{
errorCount++;
if (logger.isTraceEnabled())
{
"not agree with the actual number of entry " +
}
}
}
finally
{
}
}
/**
* Iterate through the entries in an index to perform a check for
* index cleanliness. For each ID in the index we check that the
* entry it refers to does indeed contain the expected value.
*
* @throws StorageRuntimeException If an error occurs in the JE database.
* @throws DirectoryException If an error occurs reading values in the index.
*/
{
if (verifyDN2ID)
{
}
else if (verifyID2Children)
{
}
else if (verifyID2Subtree)
{
}
{
// TODO: Need to iterate through ExtendedMatchingRules indexes.
}
{
}
}
/**
* Iterate through the entries in DN2ID to perform a check for
* index cleanliness.
*
* @throws StorageRuntimeException If an error occurs in the JE database.
*/
{
try
{
{
keyCount++;
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
try
{
}
catch (Exception e)
{
errorCount++;
logger.traceException(e);
continue;
}
{
errorCount++;
if (logger.isTraceEnabled())
{
}
}
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.trace("File dn2id has DN <%s> referencing entry with wrong DN <%s>%n", key, entry.getName());
}
}
}
}
finally
{
}
}
/**
* Iterate through the entries in ID2Children to perform a check for
* index cleanliness.
*
* @throws StorageRuntimeException If an error occurs in the JE database.
*/
{
try
{
{
keyCount++;
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
if (entryIDList.isDefined())
{
try
{
}
catch (Exception e)
{
logger.traceException(e);
errorCount++;
continue;
}
{
errorCount++;
if (logger.isTraceEnabled())
{
}
continue;
}
{
try
{
}
catch (Exception e)
{
logger.traceException(e);
errorCount++;
continue;
}
if (childEntry == null)
{
errorCount++;
if (logger.isTraceEnabled())
{
}
continue;
}
{
errorCount++;
if (logger.isTraceEnabled())
{
"referencing ID %d with non-child DN <%s>%n",
}
}
}
}
}
}
finally
{
}
}
/**
* Iterate through the entries in ID2Subtree to perform a check for
* index cleanliness.
*
* @throws StorageRuntimeException If an error occurs in the JE database.
*/
{
try
{
{
keyCount++;
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
if (entryIDList.isDefined())
{
try
{
}
catch (Exception e)
{
logger.traceException(e);
errorCount++;
continue;
}
{
errorCount++;
if (logger.isTraceEnabled())
{
}
continue;
}
{
try
{
}
catch (Exception e)
{
logger.traceException(e);
errorCount++;
continue;
}
if (subordEntry == null)
{
errorCount++;
if (logger.isTraceEnabled())
{
}
continue;
}
{
errorCount++;
if (logger.isTraceEnabled())
{
"referencing ID %d with non-subordinate DN <%s>%n",
}
}
}
}
}
}
finally
{
}
}
/**
* Increment the counter for a key that has exceeded the
* entry limit. The counter gives the number of entries that have
* referenced the key.
*
* @param index The index containing the key.
* @param key A key that has exceeded the entry limit.
*/
{
{
}
{
counter++;
}
else
{
counter = 1L;
}
}
/**
* Update the statistical information for an index record.
*
* @param entryIDSet The set of entry IDs for the index record.
*/
{
if (!entryIDSet.isDefined())
{
}
else
{
{
}
}
}
/**
* Iterate through the entries in a VLV index to perform a check for index
* cleanliness.
*
* @param vlvIndex The VLV index to perform the check against.
* @param verifyID True to verify the IDs against id2entry.
* @throws StorageRuntimeException If an error occurs in the JE database.
* @throws DirectoryException If an error occurs reading values in the index.
*/
{
{
return;
}
try
{
{
{
keyCount++;
{
// Make sure the values is larger then the previous one.
if(logger.isTraceEnabled())
{
}
errorCount++;
}
{
// If this is the last one in a bounded set, make sure it is the
// same as the database key.
{
if(logger.isTraceEnabled())
{
"index %s. Last values bytes %s, Key bytes %s",
}
errorCount++;
}
}
lastValues = values;
if(verifyID)
{
try
{
}
catch (Exception e)
{
logger.traceException(e);
errorCount++;
continue;
}
{
errorCount++;
if (logger.isTraceEnabled())
{
}
continue;
}
{
errorCount++;
if(logger.isTraceEnabled())
{
"which does not match the values%n%s",
}
}
}
}
}
}
finally
{
}
}
/**
* Iterate through the entries in an attribute index to perform a check for
* index cleanliness.
* @param index The index database to be checked.
* @throws StorageRuntimeException If an error occurs in the JE database.
*/
throws StorageRuntimeException
{
{
return;
}
try
{
{
keyCount++;
try
{
}
catch (Exception e)
{
errorCount++;
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
continue;
}
if (entryIDList.isDefined())
{
{
{
}
try
{
}
catch (Exception e)
{
logger.traceException(e);
errorCount++;
continue;
}
{
errorCount++;
if (logger.isTraceEnabled())
{
}
continue;
}
// As an optimization avoid passing in a real set and wasting time
// hashing and comparing a potentially large set of values, as well
// as using up memory. Instead just intercept the add() method and
// detect when an equivalent value has been added.
// We need to use an AtomicBoolean here since anonymous classes
// require referenced external variables to be final.
{
{
// The set is always empty.
}
public int size()
{
// The set is always empty.
return 0;
}
public boolean add(ByteString e)
{
{
// We could terminate processing at this point by throwing an
// UnsupportedOperationException, but this optimization is
// already ugly enough.
foundMatchingKey.set(true);
}
return true;
}
};
if (!foundMatchingKey.get())
{
errorCount++;
if (logger.isTraceEnabled())
{
+ "<%s> which does not match the value%n%s",
}
}
}
}
}
}
finally
{
}
}
/**
* Check that an index is complete for a given entry.
*
* @param entryID The entry ID.
* @param entry The entry to be checked.
*/
{
if (verifyDN2ID)
{
}
if (verifyID2Children)
{
}
if (verifyID2Subtree)
{
}
}
/**
* Check that the DN2ID index is complete for a given entry.
*
* @param entryID The entry ID.
* @param entry The entry to be checked.
*/
{
// Check the ID is in dn2id with the correct DN.
try
{
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
}
catch (Exception e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
// Check the parent DN is in dn2id.
{
try
{
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
}
catch (Exception e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
}
}
/**
* Check that the ID2Children index is complete for a given entry.
*
* @param entryID The entry ID.
* @param entry The entry to be checked.
*/
{
{
try
{
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
}
catch (Exception e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
{
try
{
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
{
}
}
catch (StorageRuntimeException e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
}
}
}
/**
* Check that the ID2Subtree index is complete for a given entry.
*
* @param entryID The entry ID.
* @param entry The entry to be checked.
*/
{
{
try
{
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
}
catch (Exception e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
{
try
{
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
{
}
}
catch (StorageRuntimeException e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
}
}
}
/**
* Construct a printable string from a raw key value.
*
* @param index
* The index database containing the key value.
* @param key
* The bytes of the key.
* @return A string that may be logged or printed.
*/
{
}
/**
* Construct a printable string from a raw key value.
*
* @param vlvIndex The vlvIndex database containing the key value.
* @param keySortValues THe sort values that is being used as the key.
* @return A string that may be logged or printed.
*/
{
if(keySortValues != null)
{
}
else
{
}
}
/**
* Check that an attribute index is complete for a given entry.
*
* @param entryID The entry ID.
* @param entry The entry to be checked.
*/
{
{
try
{
{
}
}
catch (DirectoryException e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
}
}
{
try
{
&& !vlvIndex.containsValues(
{
if(logger.isTraceEnabled())
{
}
errorCount++;
}
}
catch (DirectoryException e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
catch (StorageRuntimeException e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
}
}
/**
* Check that an attribute index is complete for a given attribute.
*
* @param attrIndex The attribute index to be checked.
* @param entryID The entry ID.
* @param attrList The attribute to be checked.
* @throws DirectoryException If a Directory Server error occurs.
*/
private void verifyAttribute(ReadableStorage txn, AttributeIndex attrIndex, EntryID entryID, List<Attribute> attrList)
throws DirectoryException
{
{
return;
}
// TODO: Add support for Extended Matching Rules indexes.
if (presenceIndex != null)
{
}
{
{
if (equalityIndex != null)
{
}
if (substringIndex != null)
{
{
}
}
if (orderingIndex != null)
{
}
if (approximateIndex != null)
{
}
}
}
}
{
try
{
{
if (logger.isTraceEnabled())
{
}
errorCount++;
}
{
}
}
catch (StorageRuntimeException e)
{
if (logger.isTraceEnabled())
{
logger.traceException(e);
}
errorCount++;
}
}
{
try
{
}
catch (DecodeException e)
{
e.getMessageObject(), e);
}
}
/**
* Get the parent DN of a given DN.
*
* @param dn The DN.
* @return The parent DN or null if the given DN is a base DN.
*/
{
{
return null;
}
return dn.getParentDNInSuffix();
}
/** This class reports progress of the verify job at fixed intervals. */
private final class ProgressTask extends TimerTask
{
/** The total number of records to process. */
private long totalCount;
/**
* The number of records that had been processed at the time of the
* previous progress report.
*/
private long previousCount;
/** The time in milliseconds of the previous progress report. */
private long previousTime;
/**
* The number of bytes in a megabyte.
* Note that 1024*1024 bytes may eventually become known as a mebibyte(MiB).
*/
/**
* Create a new verify progress task.
* @param indexIterator boolean, indicates if the task is iterating
* through indexes or the entries.
* @throws StorageRuntimeException An error occurred while accessing the JE
* database.
*/
{
if (indexIterator)
{
if (verifyDN2ID)
{
}
else if (verifyID2Children)
{
}
else if (verifyID2Subtree)
{
}
{
totalCount = 0;
// TODO: Add support for Extended Matching Rules indexes.
}
{
}
}
else
{
}
}
{
{
}
return 0;
}
/** The action to be performed by this timer task. */
public void run()
{
long latestCount = keyCount;
if (deltaTime == 0)
{
return;
}
try
{
// FIXME JNR compute the cache miss rate
float cacheMissRate = 0;
}
catch (StorageRuntimeException e)
{
logger.traceException(e);
}
}
}
}