IndexDatabase.java revision 1461
260N/A/*
260N/A * CDDL HEADER START
260N/A *
260N/A * The contents of this file are subject to the terms of the
260N/A * Common Development and Distribution License (the "License").
260N/A * You may not use this file except in compliance with the License.
260N/A *
260N/A * See LICENSE.txt included in this distribution for the specific
260N/A * language governing permissions and limitations under the License.
260N/A *
260N/A * When distributing Covered Code, include this CDDL HEADER in each
260N/A * file and include the License file at LICENSE.txt.
260N/A * If applicable, add the following below this CDDL HEADER, with the
260N/A * fields enclosed by brackets "[]" replaced with your own identifying
260N/A * information: Portions Copyright [yyyy] [name of copyright owner]
260N/A *
260N/A * CDDL HEADER END
260N/A */
260N/A
260N/A/*
260N/A * Copyright (c) 2008, 2012, Oracle and/or its affiliates. All rights reserved.
260N/A */
260N/A
260N/Apackage org.opensolaris.opengrok.index;
260N/A
260N/Aimport java.io.BufferedInputStream;
260N/Aimport java.io.File;
260N/Aimport java.io.FileInputStream;
260N/Aimport java.io.FileNotFoundException;
260N/Aimport java.io.IOException;
260N/Aimport java.io.InputStream;
260N/Aimport java.util.ArrayList;
260N/Aimport java.util.Arrays;
260N/Aimport java.util.Comparator;
260N/Aimport java.util.List;
260N/Aimport java.util.concurrent.ExecutorService;
260N/Aimport java.util.logging.Level;
260N/Aimport java.util.logging.Logger;
260N/A
260N/Aimport org.apache.lucene.analysis.Analyzer;
260N/Aimport org.apache.lucene.analysis.standard.StandardAnalyzer;
260N/Aimport org.apache.lucene.document.DateTools;
260N/Aimport org.apache.lucene.document.Document;
260N/Aimport org.apache.lucene.document.Fieldable;
260N/Aimport org.apache.lucene.index.IndexReader;
260N/Aimport org.apache.lucene.index.IndexWriter;
260N/Aimport org.apache.lucene.index.IndexWriterConfig;
260N/Aimport org.apache.lucene.index.IndexWriterConfig.OpenMode;
260N/Aimport org.apache.lucene.index.Term;
260N/Aimport org.apache.lucene.index.TermEnum;
260N/Aimport org.apache.lucene.queryParser.ParseException;
260N/Aimport org.apache.lucene.search.IndexSearcher;
260N/Aimport org.apache.lucene.search.Query;
260N/Aimport org.apache.lucene.search.TopDocs;
260N/Aimport org.apache.lucene.search.spell.LuceneDictionary;
260N/Aimport org.apache.lucene.search.spell.SpellChecker;
260N/Aimport org.apache.lucene.store.FSDirectory;
260N/Aimport org.apache.lucene.store.LockFactory;
260N/Aimport org.apache.lucene.store.NoLockFactory;
260N/Aimport org.apache.lucene.store.SimpleFSLockFactory;
260N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
260N/Aimport org.opensolaris.opengrok.analysis.Ctags;
260N/Aimport org.opensolaris.opengrok.analysis.Definitions;
260N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer;
260N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
260N/Aimport org.opensolaris.opengrok.configuration.Project;
260N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
260N/Aimport org.opensolaris.opengrok.history.HistoryException;
260N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
260N/Aimport org.opensolaris.opengrok.search.QueryBuilder;
260N/Aimport org.opensolaris.opengrok.search.SearchEngine;
260N/Aimport org.opensolaris.opengrok.util.IOUtils;
260N/Aimport org.opensolaris.opengrok.web.Util;
260N/A
260N/A/**
260N/A * This class is used to create / update the index databases. Currently we use
260N/A * one index database per project.
260N/A *
260N/A * @author Trond Norbye
260N/A * @author Lubos Kosco , update for lucene 3.0.0
260N/A */
260N/Apublic class IndexDatabase {
260N/A
260N/A private Project project;
260N/A private FSDirectory indexDirectory;
260N/A private FSDirectory spellDirectory;
260N/A private IndexWriter writer;
260N/A private TermEnum uidIter;
260N/A private IgnoredNames ignoredNames;
260N/A private Filter includedNames;
260N/A private AnalyzerGuru analyzerGuru;
260N/A private File xrefDir;
260N/A private boolean interrupted;
260N/A private List<IndexChangedListener> listeners;
260N/A private File dirtyFile;
260N/A private final Object lock = new Object();
260N/A private boolean dirty;
260N/A private boolean running;
260N/A private List<String> directories;
260N/A static final Logger logger = Logger.getLogger(IndexDatabase.class.getName());
260N/A private Ctags ctags;
260N/A private LockFactory lockfact;
260N/A
260N/A /**
260N/A * Create a new instance of the Index Database. Use this constructor if
260N/A * you don't use any projects
260N/A *
260N/A * @throws java.io.IOException if an error occurs while creating directories
260N/A */
260N/A public IndexDatabase() throws IOException {
260N/A this(null);
260N/A }
260N/A
260N/A /**
260N/A * Create a new instance of an Index Database for a given project
260N/A * @param project the project to create the database for
260N/A * @throws java.io.IOException if an errror occurs while creating directories
260N/A */
260N/A public IndexDatabase(Project project) throws IOException {
260N/A this.project = project;
260N/A lockfact = new SimpleFSLockFactory();
260N/A initialize();
260N/A }
260N/A
260N/A /**
260N/A * Update the index database for all of the projects. Print progress to
260N/A * standard out.
260N/A * @param executor An executor to run the job
260N/A * @throws IOException if an error occurs
260N/A */
260N/A public static void updateAll(ExecutorService executor) throws IOException {
260N/A updateAll(executor, null);
260N/A }
260N/A
260N/A /**
260N/A * Update the index database for all of the projects
260N/A * @param executor An executor to run the job
260N/A * @param listener where to signal the changes to the database
260N/A * @throws IOException if an error occurs
260N/A */
260N/A static void updateAll(ExecutorService executor, IndexChangedListener listener) throws IOException {
260N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
260N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
260N/A
260N/A if (env.hasProjects()) {
260N/A for (Project project : env.getProjects()) {
260N/A dbs.add(new IndexDatabase(project));
260N/A }
260N/A } else {
260N/A dbs.add(new IndexDatabase());
260N/A }
260N/A
260N/A for (IndexDatabase d : dbs) {
260N/A final IndexDatabase db = d;
260N/A if (listener != null) {
260N/A db.addIndexChangedListener(listener);
260N/A }
260N/A
260N/A executor.submit(new Runnable() {
260N/A
260N/A @Override
260N/A public void run() {
260N/A try {
260N/A db.update();
260N/A } catch (Throwable e) {
260N/A logger.warning("Problem updating " + db + ": "+ e.getMessage());
260N/A logger.log(Level.FINE, "updateAll", e);
260N/A }
260N/A }
260N/A });
260N/A }
260N/A }
260N/A
260N/A /**
260N/A * Update the index database for a number of sub-directories
260N/A * @param executor An executor to run the job
260N/A * @param listener where to signal the changes to the database
260N/A * @param paths
260N/A */
260N/A public static void update(ExecutorService executor, IndexChangedListener listener, List<String> paths) {
260N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
260N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
260N/A
260N/A for (String path : paths) {
260N/A Project project = Project.getProject(path);
260N/A if (project == null && env.hasProjects()) {
260N/A logger.warning("Could not find a project for '" + path + "'");
260N/A } else {
260N/A IndexDatabase db = null;
260N/A
260N/A try {
260N/A if (project == null) {
260N/A db = new IndexDatabase();
260N/A } else {
260N/A db = new IndexDatabase(project);
260N/A }
260N/A
int idx = dbs.indexOf(db);
if (idx != -1) {
db = dbs.get(idx);
}
if (db.addDirectory(path)) {
if (idx == -1) {
dbs.add(db);
}
} else {
logger.warning("Directory does not exist '" + path + "'");
}
} catch (IOException e) {
logger.warning("An error occured while updating "
+ db + ": " + e.getMessage());
logger.log(Level.FINE, "update", e);
}
}
for (final IndexDatabase db : dbs) {
db.addIndexChangedListener(listener);
executor.submit(new Runnable() {
@Override
public void run() {
try {
db.update();
} catch (Throwable e) {
logger.warning("An error occured while updating "
+ db + ": " + e.getLocalizedMessage());
logger.log(Level.FINE, "run", e);
}
}
});
}
}
}
@SuppressWarnings("PMD.CollapsibleIfStatements")
private void initialize() throws IOException {
synchronized (this) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
File spellDir = new File(env.getDataRootFile(), "spellIndex");
if (project != null) {
indexDir = new File(indexDir, project.getPath());
spellDir = new File(spellDir, project.getPath());
}
if (!indexDir.exists() && !indexDir.mkdirs()) {
// to avoid race conditions, just recheck..
if (!indexDir.exists()) {
throw new FileNotFoundException("Failed to create root directory '"
+ indexDir.getAbsolutePath() + "'");
}
}
if (!spellDir.exists() && !spellDir.mkdirs()) {
if (!spellDir.exists()) {
throw new FileNotFoundException("Failed to create root directory '"
+ spellDir.getAbsolutePath() + "'");
}
}
if (!env.isUsingLuceneLocking()) {
lockfact = NoLockFactory.getNoLockFactory();
}
indexDirectory = FSDirectory.open(indexDir,lockfact);
spellDirectory = FSDirectory.open(spellDir,lockfact);
ignoredNames = env.getIgnoredNames();
includedNames = env.getIncludedNames();
analyzerGuru = new AnalyzerGuru();
if (env.isGenerateHtml()) {
xrefDir = new File(env.getDataRootFile(), "xref");
}
listeners = new ArrayList<IndexChangedListener>();
dirtyFile = new File(indexDir, "dirty");
dirty = dirtyFile.exists();
directories = new ArrayList<String>();
}
}
/**
* By default the indexer will traverse all directories in the project.
* If you add directories with this function update will just process
* the specified directories.
*
* @param dir The directory to scan
* @return <code>true</code> if the file is added, false oth
*/
@SuppressWarnings("PMD.UseStringBufferForStringAppends")
public boolean addDirectory(String dir) {
String directory = dir;
if (directory.startsWith("\\")) {
directory = directory.replace('\\', '/');
} else if (directory.charAt(0) != '/') {
directory = "/" + directory;
}
File file = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), directory);
if (file.exists()) {
directories.add(directory);
return true;
}
return false;
}
/**
* Update the content of this index database
* @throws IOException if an error occurs
* @throws HistoryException if an error occurs when accessing the history
*/
@SuppressWarnings({ "resource", "boxing" })
public void update() throws IOException, HistoryException {
synchronized (lock) {
if (running) {
throw new IOException("Indexer already running");
}
running = true;
interrupted = false;
}
String ctgs = RuntimeEnvironment.getInstance().getCtags();
if (ctgs != null) {
ctags = new Ctags();
ctags.setBinary(ctgs);
ctags.setOptionsFile(RuntimeEnvironment.getInstance()
.getCtagsOptionsFile());
}
if (ctags == null) {
logger.warning("Unable to run ctags! Searching definitions will not work!");
}
Analyzer analyzer = null;
try {
//TODO we might need to add writer.commit after certain phases of
// index generation, right now it will only happen in the end
analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
//iwc.setRAMBufferSizeMB(256.0); //TODO check what is the sweet spot
writer = new IndexWriter(indexDirectory, iwc);
writer.commit(); // to make sure index exists on the disk
//writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
if (directories.isEmpty()) {
if (project == null) {
directories.add("");
} else {
directories.add(project.getPath());
}
}
for (String dir : directories) {
File sourceRoot;
if ("".equals(dir)) {
sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
} else {
sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
}
HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
String startuid = Util.path2uid(dir, "");
IndexReader reader = IndexReader.open(indexDirectory); // open existing index
try {
uidIter = reader.terms(new Term("u", startuid)); // init uid iterator
//TODO below should be optional, since it traverses the tree once more to get total count! :(
int file_cnt = 0;
if (RuntimeEnvironment.getInstance().isPrintProgress()) {
logger.log(Level.INFO, "Counting files in ''{0}'' ...", dir);
file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
if (logger.isLoggable(Level.INFO)) {
logger.log(Level.INFO, "Need to process {0} files for ''{1}''",
new Object[] { file_cnt, dir });
}
}
indexDown(sourceRoot, dir, false, 0, file_cnt);
while (uidIter.term() != null && uidIter.term().field().equals("u")
&& uidIter.term().text().startsWith(startuid))
{
removeFile();
uidIter.next();
}
} finally {
reader.close();
}
}
} finally {
IOUtils.close(writer);
if (ctags != null) {
ctags.close();
}
IOUtils.close(analyzer);
synchronized (lock) {
running = false;
}
}
if (!isInterrupted() && isDirty()) {
if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
optimize();
}
createSpellingSuggestions();
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File timestamp = new File(env.getDataRootFile(), "timestamp");
if (timestamp.exists()) {
if (!timestamp.setLastModified(System.currentTimeMillis())) {
logger.warning("Failed to set last modified time on '"
+ timestamp.getAbsolutePath()
+ "'used for timestamping the index database");
}
} else if (!timestamp.createNewFile()) {
logger.warning("Failed to create file '"
+ timestamp.getAbsolutePath()
+ "', used for timestamping the index database");
}
}
}
/**
* Optimize all index databases
* @param executor An executor to run the job
* @throws IOException if an error occurs
*/
static void optimizeAll(ExecutorService executor) throws IOException {
List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
for (Project project : env.getProjects()) {
dbs.add(new IndexDatabase(project));
}
} else {
dbs.add(new IndexDatabase());
}
for (IndexDatabase d : dbs) {
final IndexDatabase db = d;
if (db.isDirty()) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
db.update();
} catch (Throwable e) {
logger.warning("Problem updating " + db + ": "
+ e.getMessage());
logger.log(Level.FINE, "optimizeAll", e);
}
}
});
}
}
}
/**
* Optimize the index database
*/
public void optimize() {
synchronized (lock) {
if (running) {
logger.warning("Optimize terminated... Someone else is updating / optimizing it!");
return ;
}
running = true;
}
@SuppressWarnings("resource")
IndexWriter wrt = null;
@SuppressWarnings("resource")
Analyzer analyzer = null;
try {
logger.info("Optimizing " + this + " ...");
analyzer = new StandardAnalyzer(SearchEngine.LUCENE_VERSION);
IndexWriterConfig conf =
new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
wrt = new IndexWriter(indexDirectory, conf);
wrt.forceMerge(1); // this is deprecated and not needed anymore
logger.info("Optimizing " + this + " done");
synchronized (lock) {
if (dirtyFile.exists() && !dirtyFile.delete()) {
logger.log(Level.FINE, "Failed to remove \"dirty-file\" ''{0}''",
dirtyFile.getAbsolutePath());
}
dirty = false;
}
} catch (IOException e) {
logger.warning(this + " optimizing problem: " + e.getMessage());
logger.log(Level.FINE, "optimize", e);
} finally {
IOUtils.close(wrt);
IOUtils.close(analyzer);
synchronized (lock) {
running = false;
}
}
}
/**
* Generate a spelling suggestion for the definitions stored in defs
*/
@SuppressWarnings("resource")
public void createSpellingSuggestions() {
IndexReader indexReader = null;
SpellChecker checker = null;
Analyzer analyzer = null;
try {
logger.info("Generating spelling suggestions for " + this + " ...");
indexReader = IndexReader.open(indexDirectory);
checker = new SpellChecker(spellDirectory);
//TODO below seems only to index "defs" , possible bug ?
analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc =
new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
checker.indexDictionary(new LuceneDictionary(indexReader, "defs"),
iwc, false);
logger.info("Generating spelling suggestions for " + this + " done");
} catch (IOException e) {
logger.warning("Problem when generating spelling suggestions for "
+ this + ": " + e.getMessage());
logger.log(Level.FINE, "createSpellingSuggestions", e);
} finally {
IOUtils.close(indexReader);
IOUtils.close(spellDirectory);
IOUtils.close(analyzer);
}
}
private boolean isDirty() {
synchronized (lock) {
return dirty;
}
}
private void setDirty() {
synchronized (lock) {
try {
if (!dirty && !dirtyFile.createNewFile()) {
if (!dirtyFile.exists() && logger.isLoggable(Level.FINE)) {
logger.fine("Failed to create \"dirty-file\" '" +
dirtyFile.getAbsolutePath() + "'");
}
dirty = true;
}
} catch (IOException e) {
logger.log(Level.FINE,"unable to creating dirty file", e);
}
}
}
/**
* Remove a stale file (uidIter.term().text()) from the index database
* (and the xref file)
* @throws java.io.IOException if an error occurs
*/
private void removeFile() throws IOException {
String path = Util.uid2url(uidIter.term().text());
for (IndexChangedListener listener : listeners) {
listener.fileRemove(path);
}
writer.deleteDocuments(uidIter.term());
File xrefFile = new File(xrefDir, path);
File parent = xrefFile.getParentFile();
if (!xrefFile.delete() && xrefFile.exists()) {
logger.log(Level.INFO, "Failed to remove obsolete xref-file ''{0}''",
xrefFile.getAbsolutePath());
}
// Remove the parent directory if it's empty
if (parent.delete()) {
logger.log(Level.FINE, "Removed empty xref dir ''{0}''",
parent.getAbsolutePath());
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileRemoved(path);
}
}
/**
* Add a file to the Lucene index (and generate a xref file)
* @param file The file to add
* @param path The path to the file (from source root)
* @throws java.io.IOException if an error occurs
*/
@SuppressWarnings("resource")
private void addFile(File file, String path) throws IOException {
final InputStream in =
new BufferedInputStream(new FileInputStream(file));
FileAnalyzer fa = null;
try {
fa = AnalyzerGuru.getAnalyzer(in, path);
for (IndexChangedListener listener : listeners) {
listener.fileAdd(path, fa.getClass().getSimpleName());
}
fa.setCtags(ctags);
fa.setProject(Project.getProject(path));
Document d;
try {
d = analyzerGuru.getDocument(file, in, path, fa);
} catch (Exception e) {
logger.log(Level.INFO,
"Skipped file ''{0}'' because the analyzer didn''t " +
"understand it.", path);
logger.log(Level.FINE, "addFile", e);
return;
}
writer.addDocument(d, fa);
Genre g = fa.getFactory().getGenre();
if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
File xrefFile = new File(xrefDir, path);
// If mkdirs() returns false, the failure is most likely
// because the file already exists. But to check for the
// file first and only add it if it doesn't exists would
// only increase the file IO...
if (!xrefFile.getParentFile().mkdirs()) {
assert xrefFile.getParentFile().exists();
}
fa.writeXref(xrefDir, path);
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileAdded(path, fa.getClass().getSimpleName());
}
} finally {
IOUtils.close(in);
IOUtils.close(fa);
}
}
/**
* Check if I should accept this file into the index database
* @param file the file to check
* @return true if the file should be included, false otherwise
*/
private boolean accept(File file) {
if (!includedNames.isEmpty() &&
// the filter should not affect directory names
(!(file.isDirectory() || includedNames.match(file))) ) {
return false;
}
if (ignoredNames.ignore(file)) {
return false;
}
String absolutePath = file.getAbsolutePath();
if (!file.canRead()) {
logger.warning("Could not read " + absolutePath);
return false;
}
try {
String canonicalPath = file.getCanonicalPath();
if (!absolutePath.equals(canonicalPath) && !acceptSymlink(absolutePath, canonicalPath)) {
logger.log(Level.FINE, "Skipped symlink ''{0}'' -> ''{1}''",
new Object[]{absolutePath, canonicalPath});
return false;
}
//below will only let go files and directories, anything else is considered special and is not added
if (!file.isFile() && !file.isDirectory()) {
logger.warning("Ignored special file '" + absolutePath + "'");
return false;
}
} catch (IOException exp) {
logger.warning("Failed to resolve name '" + absolutePath + "'");
logger.log(Level.FINE, "accept", exp);
}
if (file.isDirectory()) {
// always accept directories so that their files can be examined
return true;
}
if (HistoryGuru.getInstance().hasHistory(file)) {
// versioned files should always be accepted
return true;
}
// this is an unversioned file, check if it should be indexed
return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
}
boolean accept(File parent, File file) {
try {
File f1 = parent.getCanonicalFile();
File f2 = file.getCanonicalFile();
if (f1.equals(f2)) {
logger.log(Level.INFO, "Skipping links to itself (''{0}'' ''{1}'')",
new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
return false;
}
// Now, let's verify that it's not a link back up the chain...
File t1 = f1;
while ((t1 = t1.getParentFile()) != null) {
if (f2.equals(t1)) {
logger.log(Level.INFO, "Skipping links to parent (''{0}'' ''{1}'')",
new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
return false;
}
}
return accept(file);
} catch (IOException ex) {
logger.log(Level.WARNING, "Failed to resolve name (''{0}'' ''{1}'')",
new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
}
return false;
}
/**
* Check if I should accept the path containing a symlink
* @param absolutePath the path with a symlink to check
* @param canonicalPath the canonical path to the file
* @return true if the file should be accepted, false otherwise
*/
private boolean acceptSymlink(String absolutePath, String canonicalPath) throws IOException {
// Always accept local symlinks
if (isLocal(canonicalPath)) {
return true;
}
for (String allowedSymlink : RuntimeEnvironment.getInstance().getAllowedSymlinks()) {
if (absolutePath.startsWith(allowedSymlink)) {
String allowedTarget = new File(allowedSymlink).getCanonicalPath();
if (canonicalPath.startsWith(allowedTarget) &&
absolutePath.substring(allowedSymlink.length()).equals(canonicalPath.substring(allowedTarget.length()))) {
return true;
}
}
}
return false;
}
/**
* Check if a file is local to the current project. If we don't have
* projects, check if the file is in the source root.
*
* @param path the path to a file
* @return true if the file is local to the current repository
*/
private boolean isLocal(String path) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String srcRoot = env.getSourceRootPath();
boolean local = false;
if (path.startsWith(srcRoot)) {
if (env.hasProjects()) {
String relPath = path.substring(srcRoot.length());
if (project.equals(Project.getProject(relPath))) {
// File is under the current project, so it's local.
local = true;
}
} else {
// File is under source root, and we don't have projects, so
// consider it local.
local = true;
}
}
return local;
}
/**
* Generate indexes recursively
* @param dir the root indexDirectory to generate indexes for
* @param path the path
* @param count_only if true will just traverse the source root and count files
* @param cur_count current count during the traversal of the tree
* @param est_total estimate total files to process
*
*/
@SuppressWarnings("boxing")
private int indexDown(File dir, String parent, boolean count_only, int cur_count, int est_total) throws IOException {
int lcur_count=cur_count;
if (isInterrupted()) {
return lcur_count;
}
if (!accept(dir)) {
return lcur_count;
}
File[] files = dir.listFiles();
if (files == null) {
logger.severe("Failed to get file listing for '" + dir.getAbsolutePath() + "'");
return lcur_count;
}
Arrays.sort(files, new Comparator<File>() {
@Override
public int compare(File p1, File p2) {
return p1.getName().compareTo(p2.getName());
}
});
for (File file : files) {
if (accept(dir, file)) {
String path = parent + '/' + file.getName();
if (file.isDirectory()) {
lcur_count = indexDown(file, path, count_only, lcur_count, est_total);
} else {
lcur_count++;
if (count_only) {
continue;
}
if (RuntimeEnvironment.getInstance().isPrintProgress()
&& est_total > 0 && logger.isLoggable(Level.INFO) )
{
logger.log(Level.INFO, "Progress: {0} ({1}%)",
new Object[] { lcur_count,
(lcur_count * 100.0f / est_total) });
}
if (uidIter != null) {
String uid = Util.path2uid(path, DateTools
.timeToString(file.lastModified(),
DateTools.Resolution.MILLISECOND)); // construct uid for doc
while (uidIter.term() != null && uidIter.term().field().equals("u") &&
uidIter.term().text().compareTo(uid) < 0) {
removeFile();
uidIter.next();
}
if (uidIter.term() != null && uidIter.term().field().equals("u") &&
uidIter.term().text().compareTo(uid) == 0) {
uidIter.next(); // keep matching docs
continue;
}
}
try {
addFile(file, path);
} catch (Exception e) {
logger.warning("Failed to add file '"
+ file.getAbsolutePath() + "': " + e.getMessage());
}
}
}
}
return lcur_count;
}
/**
* Interrupt the index generation (and the index generation will stop as
* soon as possible)
*/
public void interrupt() {
synchronized (lock) {
interrupted = true;
}
}
private boolean isInterrupted() {
synchronized (lock) {
return interrupted;
}
}
/**
* Register an object to receive events when modifications is done to the
* index database.
*
* @param listener the object to receive the events
*/
public void addIndexChangedListener(IndexChangedListener listener) {
listeners.add(listener);
}
/**
* Remove an object from the lists of objects to receive events when
* modifications is done to the index database
*
* @param listener the object to remove
*/
public void removeIndexChangedListener(IndexChangedListener listener) {
listeners.remove(listener);
}
/**
* List all files in all of the index databases
* @throws IOException if an error occurs
*/
public static void listAllFiles() throws IOException {
listAllFiles(null);
}
/**
* List all files in some of the index databases
* @param subFiles Subdirectories for the various projects to list the files
* for (or null or an empty list to dump all projects)
* @throws IOException if an error occurs
*/
public static void listAllFiles(List<String> subFiles) throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
logger.warning("Could not find a project for '"
+ path + "'");
} else {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listFiles();
}
}
/**
* List all of the files in this index database
*
* @throws IOException If an IO error occurs while reading from the database
*/
@SuppressWarnings("resource")
public void listFiles() throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory); // open existing index
iter = ireader.terms(new Term("u", "")); // init uid iterator
if (logger.isLoggable(Level.FINE)) {
while (iter.term() != null) {
logger.fine(Util.uid2url(iter.term().text()));
iter.next();
}
}
} finally {
IOUtils.close(iter);
IOUtils.close(ireader);
}
}
static void listFrequentTokens() throws IOException {
listFrequentTokens(null);
}
static void listFrequentTokens(List<String> subFiles) throws IOException {
final int limit = 4;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
logger.warning("Could not find a project for '" + path + "'");
} else {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listTokens(limit);
}
}
@SuppressWarnings("resource")
void listTokens(int freq) throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory);
iter = ireader.terms(new Term("defs", ""));
while (iter.term() != null) {
if (iter.term().field().startsWith("f")) {
if (iter.docFreq() > 16 && iter.term().text().length() > freq) {
logger.warning(iter.term().text());
}
iter.next();
} else {
break;
}
}
} finally {
IOUtils.close(iter);
IOUtils.close(ireader);
}
}
/**
* Get an indexReader for the Index database where a given file
* @param path the file to get the database for
* @return The index database where the file should be located or null if
* it cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
}
indexDir = new File(indexDir, p.getPath());
}
try {
@SuppressWarnings("resource")
FSDirectory fdir =
FSDirectory.open(indexDir,NoLockFactory.getNoLockFactory());
if (indexDir.exists() && IndexReader.indexExists(fdir)) {
ret = IndexReader.open(fdir);
}
} catch (Exception ex) {
logger.warning("Failed to open index '" + indexDir.getAbsolutePath() + "'");
logger.log(Level.FINE, "getIndexReader", ex);
}
return ret;
}
/**
* Get the latest definitions for a file from the index.
*
* @param file the file whose definitions to find
* @return definitions for the file, or {@code null} if they could not
* be found
* @throws IOException if an error happens when accessing the index
* @throws ParseException if an error happens when building the Lucene query
* @throws ClassNotFoundException if the class for the stored definitions
* instance cannot be found
*/
public static Definitions getDefinitions(File file)
throws IOException, ParseException, ClassNotFoundException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String path = env.getPathRelativeToSourceRoot(file, 0);
IndexReader ireader = getIndexReader(path);
if (ireader == null) {
// No index, no definitions...
return null;
}
try {
Query q = new QueryBuilder().setPath(path).build();
IndexSearcher searcher = new IndexSearcher(ireader);
try {
TopDocs top = searcher.search(q, 1);
if (top.totalHits == 0) {
// No hits, no definitions...
return null;
}
Document doc = searcher.doc(top.scoreDocs[0].doc);
String foundPath = doc.get("path");
// Only use the definitions if we found an exact match.
if (path.equals(foundPath)) {
Fieldable tags = doc.getFieldable("tags");
if (tags != null) {
return Definitions.deserialize(tags.getBinaryValue());
}
}
} finally {
searcher.close();
}
} finally {
ireader.close();
}
// Didn't find any definitions.
return null;
}
/**
* {@inheritDoc}
*/
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
IndexDatabase other = (IndexDatabase) obj;
return (this.project == other.project)
|| (this.project != null && this.project.equals(other.project));
}
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
int hash = 7;
hash = 41 * hash + (this.project == null ? 0 : this.project.hashCode());
return hash;
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return (project == null ? "" : project.getDescription()) + " Lucene IndexDB";
}
}