IndexDatabase.java revision 508
1056N/A/*
1056N/A * CDDL HEADER START
1276N/A *
1276N/A * The contents of this file are subject to the terms of the
1276N/A * Common Development and Distribution License (the "License").
1276N/A * You may not use this file except in compliance with the License.
1276N/A *
1276N/A * See LICENSE.txt included in this distribution for the specific
1276N/A * language governing permissions and limitations under the License.
1276N/A *
1276N/A * When distributing Covered Code, include this CDDL HEADER in each
1276N/A * file and include the License file at LICENSE.txt.
1276N/A * If applicable, add the following below this CDDL HEADER, with the
1276N/A * fields enclosed by brackets "[]" replaced with your own identifying
1276N/A * information: Portions Copyright [yyyy] [name of copyright owner]
1276N/A *
1276N/A * CDDL HEADER END
1276N/A */
1276N/A
1276N/A/*
1276N/A * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
1276N/A * Use is subject to license terms.
1276N/A */
1276N/Apackage org.opensolaris.opengrok.index;
1276N/A
1276N/Aimport java.io.BufferedInputStream;
1276N/Aimport java.io.File;
1276N/Aimport java.io.FileInputStream;
1276N/Aimport java.io.FileNotFoundException;
1276N/Aimport java.io.IOException;
1276N/Aimport java.io.InputStream;
1276N/Aimport java.util.ArrayList;
1276N/Aimport java.util.Arrays;
1276N/Aimport java.util.Comparator;
1276N/Aimport java.util.List;
1276N/Aimport java.util.concurrent.ExecutorService;
1276N/Aimport java.util.logging.Level;
1276N/Aimport java.util.logging.Logger;
1276N/Aimport org.apache.lucene.document.DateTools;
1276N/Aimport org.apache.lucene.document.Document;
1276N/Aimport org.apache.lucene.index.IndexReader;
1276N/Aimport org.apache.lucene.index.IndexWriter;
1276N/Aimport org.apache.lucene.index.Term;
1276N/Aimport org.apache.lucene.index.TermEnum;
1276N/Aimport org.apache.lucene.search.spell.LuceneDictionary;
1276N/Aimport org.apache.lucene.search.spell.SpellChecker;
1276N/Aimport org.apache.lucene.store.FSDirectory;
1276N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
1276N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer;
1276N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
1276N/Aimport org.opensolaris.opengrok.configuration.Project;
1276N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
1276N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
1276N/Aimport org.opensolaris.opengrok.web.Util;
1276N/A
1276N/A/**
1276N/A * This class is used to create / update the index databases. Currently we use
1276N/A * one index database per project.
1276N/A *
1276N/A * @author Trond Norbye
1276N/A */
1276N/Apublic class IndexDatabase {
1276N/A
1276N/A private Project project;
1276N/A private FSDirectory indexDirectory;
1276N/A private FSDirectory spellDirectory;
1276N/A private IndexWriter writer;
1276N/A private TermEnum uidIter;
1276N/A private IgnoredNames ignoredNames;
1276N/A private AnalyzerGuru analyzerGuru;
1276N/A private File xrefDir;
1276N/A private boolean interrupted;
1276N/A private List<IndexChangedListener> listeners;
1276N/A private File dirtyFile;
1276N/A private final Object lock = new Object();
1276N/A private boolean dirty;
1276N/A private boolean running;
1276N/A private List<String> directories;
1276N/A private static final Logger log = Logger.getLogger(IndexDatabase.class.getName());
1276N/A
1276N/A /**
1276N/A * Create a new instance of the Index Database. Use this constructor if
1276N/A * you don't use any projects
1276N/A *
1276N/A * @throws java.io.IOException if an error occurs while creating directories
1276N/A */
1276N/A public IndexDatabase() throws IOException {
1276N/A initialize();
1276N/A }
1276N/A
1276N/A /**
1276N/A * Create a new instance of an Index Database for a given project
1276N/A * @param project the project to create the database for
1276N/A * @throws java.io.IOException if an errror occurs while creating directories
1276N/A */
1276N/A public IndexDatabase(Project project) throws IOException {
1276N/A this.project = project;
1276N/A initialize();
1276N/A }
1276N/A
1276N/A /**
1276N/A * Update the index database for all of the projects. Print progress to
1276N/A * standard out.
1276N/A * @param executor An executor to run the job
1276N/A * @throws IOException if an error occurs
1276N/A */
1276N/A public static void updateAll(ExecutorService executor) throws IOException {
1276N/A updateAll(executor, null);
1276N/A }
1276N/A
1276N/A /**
1276N/A * Update the index database for all of the projects
1276N/A * @param executor An executor to run the job
1276N/A * @param listener where to signal the changes to the database
1276N/A * @throws IOException if an error occurs
1276N/A */
1276N/A static void updateAll(ExecutorService executor, IndexChangedListener listener) throws IOException {
1276N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
1276N/A if (env.hasProjects()) {
1276N/A for (Project project : env.getProjects()) {
1276N/A final IndexDatabase db = new IndexDatabase(project);
1276N/A if (listener != null) {
1276N/A db.addIndexChangedListener(listener);
1276N/A }
1276N/A executor.submit(new Runnable() {
1276N/A
1276N/A public void run() {
1276N/A try {
1276N/A db.update();
1276N/A } catch (Exception e) {
1276N/A log.log(Level.WARNING,"Problem updating lucene index database: ",e);
1276N/A }
1276N/A }
1276N/A });
1276N/A }
1276N/A } else {
1276N/A final IndexDatabase db = new IndexDatabase();
1276N/A if (listener != null) {
1276N/A db.addIndexChangedListener(listener);
1276N/A }
1276N/A
1276N/A executor.submit(new Runnable() {
1276N/A
1276N/A public void run() {
1276N/A try {
1276N/A db.update();
1276N/A } catch (Exception e) {
1276N/A log.log(Level.FINE,"Problem updating lucene index database: ",e);
1276N/A }
1276N/A }
1276N/A });
1276N/A }
1276N/A }
1276N/A
1276N/A @SuppressWarnings("PMD.CollapsibleIfStatements")
1276N/A private synchronized void initialize() throws IOException {
1276N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
1276N/A File indexDir = new File(env.getDataRootFile(), "index");
1276N/A File spellDir = new File(env.getDataRootFile(), "spellIndex");
1276N/A if (project != null) {
1276N/A indexDir = new File(indexDir, project.getPath());
1276N/A spellDir = new File(spellDir, project.getPath());
1276N/A }
1276N/A
1276N/A if (!indexDir.exists() && !indexDir.mkdirs()) {
1276N/A // to avoid race conditions, just recheck..
1276N/A if (!indexDir.exists()) {
1276N/A throw new FileNotFoundException("Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
1276N/A }
1276N/A }
1276N/A
1276N/A if (!spellDir.exists() && !spellDir.mkdirs()) {
1276N/A if (!spellDir.exists()) {
1276N/A throw new FileNotFoundException("Failed to create root directory [" + spellDir.getAbsolutePath() + "]");
1276N/A }
1276N/A }
1276N/A
1276N/A if (!env.isUsingLuceneLocking()) {
1276N/A FSDirectory.setDisableLocks(true);
1276N/A }
1276N/A indexDirectory = FSDirectory.getDirectory(indexDir);
1276N/A spellDirectory = FSDirectory.getDirectory(spellDir);
1276N/A ignoredNames = env.getIgnoredNames();
1276N/A analyzerGuru = new AnalyzerGuru();
1276N/A if (env.isGenerateHtml()) {
1276N/A xrefDir = new File(env.getDataRootFile(), "xref");
1276N/A }
1276N/A listeners = new ArrayList<IndexChangedListener>();
1276N/A dirtyFile = new File(indexDir, "dirty");
1276N/A dirty = dirtyFile.exists();
1276N/A directories = new ArrayList<String>();
1276N/A }
1276N/A
1276N/A /**
1276N/A * By default the indexer will traverse all directories in the project.
1276N/A * If you add directories with this function update will just process
1276N/A * the specified directories.
1276N/A *
1276N/A * @param dir The directory to scan
1276N/A * @return <code>true</code> if the file is added, false oth
1276N/A */
1276N/A @SuppressWarnings("PMD.UseStringBufferForStringAppends")
1276N/A public boolean addDirectory(String dir) {
1276N/A String directory = dir;
1276N/A if (directory.startsWith("\\")) {
1276N/A directory = directory.replace('\\', '/');
1276N/A } else if (directory.charAt(0) != '/') {
1276N/A directory = "/" + directory;
1276N/A }
1276N/A File file = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), directory);
1276N/A if (file.exists()) {
1276N/A directories.add(directory);
1276N/A return true;
1276N/A } else {
1276N/A return false;
1276N/A }
1276N/A }
1276N/A
1276N/A /**
1276N/A * Update the content of this index database
1276N/A * @throws IOException if an error occurs
1276N/A */
1276N/A public void update() throws IOException {
1276N/A synchronized (lock) {
1276N/A if (running) {
1276N/A throw new IOException("Indexer already running!");
1276N/A }
1276N/A running = true;
1276N/A interrupted = false;
1276N/A }
1276N/A try {
1276N/A writer = new IndexWriter(indexDirectory, AnalyzerGuru.getAnalyzer());
1276N/A writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
1276N/A
1276N/A if (directories.isEmpty()) {
1276N/A if (project == null) {
1276N/A directories.add("");
1276N/A } else {
1276N/A directories.add(project.getPath());
1276N/A }
1276N/A }
1276N/A
1276N/A for (String dir : directories) {
1276N/A File sourceRoot;
1276N/A if ("".equals(dir)) {
1276N/A sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
1276N/A } else {
1276N/A sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
1276N/A }
1276N/A
1276N/A String startuid = Util.uid(dir, "");
1276N/A IndexReader reader = IndexReader.open(indexDirectory); // open existing index
1276N/A try {
1276N/A uidIter = reader.terms(new Term("u", startuid)); // init uid iterator
1276N/A
1276N/A indexDown(sourceRoot, dir);
1276N/A
1276N/A while (uidIter.term() != null && uidIter.term().field().equals("u") && uidIter.term().text().startsWith(startuid)) {
1276N/A removeFile();
1276N/A uidIter.next();
1276N/A }
1276N/A } finally {
1276N/A reader.close();
1276N/A }
1276N/A }
1276N/A } finally {
1276N/A if (writer != null) {
1276N/A try {
1276N/A writer.close();
1276N/A } catch (IOException e) {
1276N/A log.log(Level.WARNING, "An error occured while closing writer", e);
1276N/A }
1276N/A }
1276N/A synchronized (lock) {
1276N/A running = false;
1276N/A }
1276N/A }
1276N/A
1276N/A if (!isInterrupted() && isDirty()) {
1276N/A if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
1276N/A optimize();
1276N/A }
1276N/A createSpellingSuggestions();
1276N/A }
1276N/A }
1276N/A
1276N/A /**
1276N/A * Optimize all index databases
1276N/A * @param executor An executor to run the job
1276N/A * @throws IOException if an error occurs
1276N/A */
1276N/A static void optimizeAll(ExecutorService executor) throws IOException {
1276N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
1276N/A if (env.hasProjects()) {
1276N/A for (Project project : env.getProjects()) {
1276N/A final IndexDatabase db = new IndexDatabase(project);
1276N/A if (db.isDirty()) {
1276N/A executor.submit(new Runnable() {
1276N/A
1276N/A public void run() {
1276N/A db.optimize();
1276N/A }
1276N/A });
1276N/A }
1276N/A }
1276N/A } else {
1056N/A final IndexDatabase db = new IndexDatabase();
1056N/A if (db.isDirty()) {
1056N/A executor.submit(new Runnable() {
1276N/A
1056N/A public void run() {
1056N/A try {
1056N/A db.update();
1276N/A } catch (IOException e) {
1056N/A log.log(Level.FINE,"Problem updating lucene index database: ",e);
1056N/A }
1056N/A }
1056N/A });
1056N/A }
1056N/A }
1056N/A }
1056N/A
1276N/A /**
1056N/A * Optimize the index database
1056N/A */
1056N/A public void optimize() {
1056N/A synchronized (lock) {
1056N/A if (running) {
1056N/A log.warning("Optimize terminated... Someone else is updating / optimizing it!");
1056N/A return ;
1276N/A }
1056N/A running = true;
1056N/A }
1056N/A IndexWriter wrt = null;
1056N/A try {
1056N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
1056N/A log.info("Optimizing the index ... ");
1056N/A }
1056N/A wrt = new IndexWriter(indexDirectory, null, false);
1276N/A wrt.optimize();
1056N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
1056N/A log.info("done");
1056N/A }
1056N/A synchronized (lock) {
1056N/A if (dirtyFile.exists() && !dirtyFile.delete()) {
1056N/A log.fine("Failed to remove \"dirty-file\": " +
1056N/A dirtyFile.getAbsolutePath());
1056N/A }
1056N/A dirty = false;
1056N/A }
1056N/A } catch (IOException e) {
1056N/A log.severe("ERROR: optimizing index: " + e);
1056N/A } finally {
1056N/A if (wrt != null) {
1056N/A try {
1056N/A wrt.close();
1056N/A } catch (IOException e) {
1056N/A log.log(Level.WARNING, "An error occured while closing writer", e);
1056N/A }
1056N/A }
1276N/A synchronized (lock) {
1056N/A running = false;
1056N/A }
1056N/A }
1056N/A }
1056N/A
1056N/A /**
1056N/A * Generate a spelling suggestion for the definitions stored in defs
1056N/A */
1056N/A public void createSpellingSuggestions() {
1056N/A IndexReader indexReader = null;
1056N/A SpellChecker checker = null;
1056N/A
1056N/A try {
1276N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
1056N/A log.info("Generating spelling suggestion index ... ");
1056N/A }
1056N/A indexReader = IndexReader.open(indexDirectory);
1056N/A checker = new SpellChecker(spellDirectory);
1276N/A checker.indexDictionary(new LuceneDictionary(indexReader, "defs"));
1056N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
1056N/A log.info("done");
1056N/A }
1056N/A } catch (IOException e) {
1276N/A log.severe("ERROR: Generating spelling: " + e);
1056N/A } finally {
1276N/A if (indexReader != null) {
1056N/A try {
1056N/A indexReader.close();
1056N/A } catch (IOException e) {
1056N/A log.log(Level.WARNING, "An error occured while closing reader", e);
1056N/A }
1056N/A }
1276N/A if (spellDirectory != null) {
1056N/A spellDirectory.close();
1276N/A }
1056N/A }
1276N/A }
1056N/A
1276N/A private boolean isDirty() {
1056N/A synchronized (lock) {
1276N/A return dirty;
1056N/A }
1276N/A }
1056N/A
1056N/A private void setDirty() {
1056N/A synchronized (lock) {
1276N/A try {
1056N/A if (!dirty && !dirtyFile.createNewFile()) {
1056N/A if (!dirtyFile.exists()) {
1276N/A log.log(Level.FINE, "Failed to create \"dirty-file\": ", dirtyFile.getAbsolutePath());
1276N/A }
1056N/A dirty = true;
1056N/A }
1056N/A } catch (IOException e) {
1056N/A log.log(Level.FINE,"When creating dirty file: ",e);
1056N/A }
1056N/A }
1056N/A }
1056N/A /**
1276N/A * Remove a stale file (uidIter.term().text()) from the index database
1056N/A * (and the xref file)
1276N/A * @throws java.io.IOException if an error occurs
1056N/A */
1276N/A private void removeFile() throws IOException {
1056N/A String path = Util.uid2url(uidIter.term().text());
1056N/A
1276N/A for (IndexChangedListener listener : listeners) {
1056N/A listener.fileRemoved(path);
1276N/A }
1056N/A writer.deleteDocuments(uidIter.term());
1276N/A
1056N/A File xrefFile = new File(xrefDir, path);
1056N/A File parent = xrefFile.getParentFile();
1276N/A
1056N/A if (!xrefFile.delete()) {
1056N/A log.info("Failed to remove obsolete xref-file: " +
1056N/A xrefFile.getAbsolutePath());
1056N/A }
1056N/A
1056N/A // Remove the parent directory if it's empty
1056N/A if (parent.delete()) {
1056N/A log.fine("Removed empty xref dir:" + parent.getAbsolutePath());
1056N/A }
1056N/A
1056N/A setDirty();
1056N/A }
1056N/A
1056N/A /**
1056N/A * Add a file to the Lucene index (and generate a xref file)
1056N/A * @param file The file to add
1056N/A * @param path The path to the file (from source root)
1056N/A * @throws java.io.IOException if an error occurs
1056N/A */
1056N/A private void addFile(File file, String path) throws IOException {
1056N/A InputStream in;
1056N/A try {
1056N/A in = new BufferedInputStream(new FileInputStream(file));
1056N/A } catch (IOException ex) {
1056N/A log.warning("Warning: " + ex.getMessage());
1056N/A return;
1056N/A }
1056N/A FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path);
1056N/A
1056N/A for (IndexChangedListener listener : listeners) {
1056N/A listener.fileAdded(path, fa.getClass().getSimpleName());
1056N/A }
1056N/A
1056N/A Document d = analyzerGuru.getDocument(file, in, path, fa);
1056N/A if (d == null) {
log.warning("Warning: did not add " + path);
} else {
writer.addDocument(d, fa);
Genre g = fa.getFactory().getGenre();
if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
File xrefFile = new File(xrefDir, path);
// If mkdirs() returns false, the failure is most likely
// because the file already exists. But to check for the
// file first and only add it if it doesn't exists would
// only increase the file IO...
if (!xrefFile.getParentFile().mkdirs()) {
assert xrefFile.getParentFile().exists();
}
fa.writeXref(xrefDir, path);
}
setDirty();
}
try {
in.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing stream", e);
}
}
/**
* Check if I should accept this file into the index database
* @param file the file to check
* @return true if the file should be included, false otherwise
*/
private boolean accept(File file) {
if (ignoredNames.ignore(file)) {
return false;
}
if (!file.canRead()) {
log.warning("Warning: could not read " + file.getAbsolutePath());
return false;
}
try {
if (!file.getAbsolutePath().equals(file.getCanonicalPath())) {
if (file.getParentFile().equals(file.getCanonicalFile().getParentFile())) {
// Lets support symlinks within the same directory, this
// should probably be extended to within the same repository
return true;
} else {
log.warning("Warning: ignored non-local symlink " + file.getAbsolutePath() +
" -> " + file.getCanonicalPath());
return false;
}
}
} catch (IOException exp) {
log.warning("Warning: Failed to resolve name: " + file.getAbsolutePath());
log.log(Level.FINE,"Stack Trace: ",exp);
}
if (file.isDirectory()) {
// always accept directories so that their files can be examined
return true;
}
if (HistoryGuru.getInstance().hasHistory(file)) {
// versioned files should always be accepted
return true;
}
// this is an unversioned file, check if it should be indexed
return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
}
/**
* Generate indexes recursively
* @param dir the root indexDirectory to generate indexes for
* @param path the path
*/
private void indexDown(File dir, String parent) throws IOException {
if (isInterrupted()) {
return;
}
if (!accept(dir)) {
return;
}
File[] files = dir.listFiles();
if (files == null) {
log.severe("Failed to get file listing for: " + dir.getAbsolutePath());
return;
}
Arrays.sort(files, new Comparator<File>() {
public int compare(File p1, File p2) {
return p1.getName().compareTo(p2.getName());
}
});
for (File file : files) {
if (accept(file)) {
String path = parent + '/' + file.getName();
if (file.isDirectory()) {
indexDown(file, path);
} else {
if (uidIter == null) {
addFile(file, path);
} else {
String uid = Util.uid(path, DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND)); // construct uid for doc
while (uidIter.term() != null && uidIter.term().field().equals("u") &&
uidIter.term().text().compareTo(uid) < 0) {
removeFile();
uidIter.next();
}
if (uidIter.term() != null && uidIter.term().field().equals("u") &&
uidIter.term().text().compareTo(uid) == 0) {
uidIter.next(); // keep matching docs
} else {
addFile(file, path);
}
}
}
}
}
}
/**
* Interrupt the index generation (and the index generation will stop as
* soon as possible)
*/
public void interrupt() {
synchronized (lock) {
interrupted = true;
}
}
private boolean isInterrupted() {
synchronized (lock) {
return interrupted;
}
}
/**
* Register an object to receive events when modifications is done to the
* index database.
*
* @param listener the object to receive the events
*/
public void addIndexChangedListener(IndexChangedListener listener) {
listeners.add(listener);
}
/**
* Remove an object from the lists of objects to receive events when
* modifications is done to the index database
*
* @param listener the object to remove
*/
public void removeIndexChangedListener(IndexChangedListener listener) {
listeners.remove(listener);
}
/**
* List all files in all of the index databases
* @throws IOException if an error occurs
*/
public static void listAllFiles() throws IOException {
listAllFiles(null);
}
/**
* List all files in some of the index databases
* @param subFiles Subdirectories for the various projects to list the files
* for (or null or an empty list to dump all projects)
* @throws IOException if an error occurs
*/
public static void listAllFiles(List<String> subFiles) throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
log.warning("Warning: Could not find a project for \"" + path + "\"");
} else {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listFiles();
}
}
/**
* List all of the files in this index database
*
* @throws IOException If an IO error occurs while reading from the database
*/
public void listFiles() throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory); // open existing index
iter = ireader.terms(new Term("u", "")); // init uid iterator
while (iter.term() != null) {
log.info(Util.uid2url(iter.term().text()));
iter.next();
}
} finally {
if (iter != null) {
try {
iter.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
if (ireader != null) {
try {
ireader.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
}
}
static void listFrequentTokens() throws IOException {
listFrequentTokens(null);
}
static void listFrequentTokens(List<String> subFiles) throws IOException {
final int limit = 4;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
log.warning("Warning: Could not find a project for \"" + path + "\"");
} else {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listTokens(limit);
}
}
public void listTokens(int freq) throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory);
iter = ireader.terms(new Term("defs", ""));
while (iter.term() != null) {
if (iter.term().field().startsWith("f")) {
if (iter.docFreq() > 16 && iter.term().text().length() > freq) {
log.warning(iter.term().text());
}
iter.next();
} else {
break;
}
}
} finally {
if (iter != null) {
try {
iter.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
if (ireader != null) {
try {
ireader.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
}
}
/**
* Get an indexReader for the Index database where a given file
* @param path the file to get the database for
* @return The index database where the file should be located or null if
* it cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
} else {
indexDir = new File(indexDir, p.getPath());
}
}
if (indexDir.exists() && IndexReader.indexExists(indexDir)) {
try {
ret = IndexReader.open(indexDir);
} catch (Exception ex) {
log.severe("Failed to open index: " + indexDir.getAbsolutePath());
log.log(Level.FINE,"Stack Trace: ",ex);
}
}
return ret;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final IndexDatabase other = (IndexDatabase) obj;
if (this.project != other.project && (this.project == null || !this.project.equals(other.project))) {
return false;
}
return true;
}
@Override
public int hashCode() {
int hash = 7;
hash = 41 * hash + (this.project == null ? 0 : this.project.hashCode());
return hash;
}
}