IndexDatabase.java revision 830
207N/A/*
207N/A * CDDL HEADER START
207N/A *
207N/A * The contents of this file are subject to the terms of the
207N/A * Common Development and Distribution License (the "License").
207N/A * You may not use this file except in compliance with the License.
207N/A *
207N/A * See LICENSE.txt included in this distribution for the specific
207N/A * language governing permissions and limitations under the License.
207N/A *
207N/A * When distributing Covered Code, include this CDDL HEADER in each
207N/A * file and include the License file at LICENSE.txt.
207N/A * If applicable, add the following below this CDDL HEADER, with the
207N/A * fields enclosed by brackets "[]" replaced with your own identifying
207N/A * information: Portions Copyright [yyyy] [name of copyright owner]
207N/A *
207N/A * CDDL HEADER END
207N/A */
207N/A
207N/A/*
1393N/A * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
207N/A * Use is subject to license terms.
1051N/A */
207N/Apackage org.opensolaris.opengrok.index;
207N/A
207N/Aimport java.io.BufferedInputStream;
207N/Aimport java.io.File;
207N/Aimport java.io.FileInputStream;
207N/Aimport java.io.FileNotFoundException;
207N/Aimport java.io.IOException;
207N/Aimport java.io.InputStream;
207N/Aimport java.util.ArrayList;
207N/Aimport java.util.Arrays;
282N/Aimport java.util.Comparator;
207N/Aimport java.util.List;
261N/Aimport java.util.concurrent.ExecutorService;
320N/Aimport java.util.logging.Level;
312N/Aimport java.util.logging.Logger;
1327N/Aimport org.apache.lucene.document.DateTools;
1318N/Aimport org.apache.lucene.document.Document;
1318N/Aimport org.apache.lucene.index.IndexReader;
207N/Aimport org.apache.lucene.index.IndexWriter;
207N/Aimport org.apache.lucene.index.Term;
1127N/Aimport org.apache.lucene.index.TermEnum;
1327N/Aimport org.apache.lucene.search.spell.LuceneDictionary;
1327N/Aimport org.apache.lucene.search.spell.SpellChecker;
1327N/Aimport org.apache.lucene.store.FSDirectory;
1318N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
1327N/Aimport org.opensolaris.opengrok.analysis.Ctags;
1327N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer;
1127N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
1127N/Aimport org.opensolaris.opengrok.configuration.Project;
1127N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
1127N/Aimport org.opensolaris.opengrok.history.HistoryException;
207N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
207N/Aimport org.opensolaris.opengrok.web.Util;
207N/A
928N/A/**
928N/A * This class is used to create / update the index databases. Currently we use
928N/A * one index database per project.
207N/A *
656N/A * @author Trond Norbye
1127N/A */
207N/Apublic class IndexDatabase {
207N/A
207N/A private Project project;
207N/A private FSDirectory indexDirectory;
678N/A private FSDirectory spellDirectory;
480N/A private IndexWriter writer;
1127N/A private TermEnum uidIter;
1318N/A private IgnoredNames ignoredNames;
1327N/A private AnalyzerGuru analyzerGuru;
207N/A private File xrefDir;
207N/A private boolean interrupted;
207N/A private List<IndexChangedListener> listeners;
207N/A private File dirtyFile;
1190N/A private final Object lock = new Object();
1190N/A private boolean dirty;
207N/A private boolean running;
928N/A private List<String> directories;
207N/A private static final Logger log = Logger.getLogger(IndexDatabase.class.getName());
207N/A private Ctags ctags;
207N/A
207N/A /**
207N/A * Create a new instance of the Index Database. Use this constructor if
207N/A * you don't use any projects
207N/A *
207N/A * @throws java.io.IOException if an error occurs while creating directories
207N/A */
1026N/A public IndexDatabase() throws IOException {
207N/A initialize();
207N/A }
207N/A
207N/A /**
253N/A * Create a new instance of an Index Database for a given project
359N/A * @param project the project to create the database for
207N/A * @throws java.io.IOException if an errror occurs while creating directories
359N/A */
274N/A public IndexDatabase(Project project) throws IOException {
1327N/A this.project = project;
656N/A initialize();
928N/A }
656N/A
207N/A /**
207N/A * Update the index database for all of the projects. Print progress to
207N/A * standard out.
1190N/A * @param executor An executor to run the job
207N/A * @throws IOException if an error occurs
207N/A */
207N/A public static void updateAll(ExecutorService executor) throws IOException {
1190N/A updateAll(executor, null);
207N/A }
207N/A
207N/A /**
207N/A * Update the index database for all of the projects
207N/A * @param executor An executor to run the job
207N/A * @param listener where to signal the changes to the database
207N/A * @throws IOException if an error occurs
1190N/A */
207N/A static void updateAll(ExecutorService executor, IndexChangedListener listener) throws IOException {
928N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
207N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
207N/A
207N/A if (env.hasProjects()) {
207N/A for (Project project : env.getProjects()) {
207N/A dbs.add(new IndexDatabase(project));
207N/A }
261N/A } else {
459N/A dbs.add(new IndexDatabase());
207N/A }
459N/A
261N/A for (IndexDatabase d : dbs) {
207N/A final IndexDatabase db = d;
207N/A if (listener != null) {
207N/A db.addIndexChangedListener(listener);
207N/A }
261N/A
207N/A executor.submit(new Runnable() {
459N/A
207N/A public void run() {
312N/A try {
207N/A db.update();
564N/A } catch (Exception e) {
1190N/A log.log(Level.FINE,"Problem updating lucene index database: ",e);
207N/A }
207N/A }
564N/A });
207N/A }
207N/A }
564N/A
564N/A /**
1190N/A * Update the index database for a number of sub-directories
564N/A * @param executor An executor to run the job
564N/A * @param listener where to signal the changes to the database
207N/A * @param paths
207N/A * @throws IOException if an error occurs
207N/A */
1190N/A public static void update(ExecutorService executor, IndexChangedListener listener, List<String> paths) throws IOException {
261N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
261N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
1054N/A
261N/A for (String path : paths) {
261N/A Project project = Project.getProject(path);
261N/A if (project == null && env.hasProjects()) {
1264N/A log.warning("Could not find a project for \"" + path + "\"");
1327N/A } else {
1327N/A IndexDatabase db;
261N/A
261N/A try {
261N/A if (project == null) {
207N/A db = new IndexDatabase();
207N/A } else {
207N/A db = new IndexDatabase(project);
668N/A }
668N/A
668N/A int idx = dbs.indexOf(db);
668N/A if (idx != -1) {
668N/A db = dbs.get(idx);
668N/A }
1327N/A
668N/A if (db.addDirectory(path)) {
668N/A if (idx == -1) {
668N/A dbs.add(db);
668N/A }
668N/A } else {
668N/A log.warning("Directory does not exist \"" + path + "\"");
1327N/A }
668N/A } catch (IOException e) {
1327N/A log.log(Level.WARNING, "An error occured while updating index", e);
668N/A
668N/A }
668N/A }
668N/A
668N/A for (final IndexDatabase db : dbs) {
668N/A db.addIndexChangedListener(listener);
668N/A executor.submit(new Runnable() {
668N/A
668N/A public void run() {
668N/A try {
668N/A db.update();
668N/A } catch (Exception e) {
668N/A log.log(Level.WARNING, "An error occured while updating index", e);
668N/A }
668N/A }
668N/A });
668N/A }
668N/A }
1327N/A }
668N/A
668N/A @SuppressWarnings("PMD.CollapsibleIfStatements")
1327N/A private void initialize() throws IOException {
1327N/A synchronized (this) {
1327N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
668N/A File indexDir = new File(env.getDataRootFile(), "index");
668N/A File spellDir = new File(env.getDataRootFile(), "spellIndex");
668N/A if (project != null) {
668N/A indexDir = new File(indexDir, project.getPath());
668N/A spellDir = new File(spellDir, project.getPath());
668N/A }
668N/A
1054N/A if (!indexDir.exists() && !indexDir.mkdirs()) {
668N/A // to avoid race conditions, just recheck..
668N/A if (!indexDir.exists()) {
668N/A throw new FileNotFoundException("Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
1264N/A }
1327N/A }
1327N/A
1327N/A if (!spellDir.exists() && !spellDir.mkdirs()) {
668N/A if (!spellDir.exists()) {
668N/A throw new FileNotFoundException("Failed to create root directory [" + spellDir.getAbsolutePath() + "]");
668N/A }
668N/A }
668N/A
668N/A if (!env.isUsingLuceneLocking()) {
668N/A FSDirectory.setDisableLocks(true);
460N/A }
580N/A indexDirectory = FSDirectory.getDirectory(indexDir);
580N/A spellDirectory = FSDirectory.getDirectory(spellDir);
580N/A ignoredNames = env.getIgnoredNames();
580N/A analyzerGuru = new AnalyzerGuru();
580N/A if (env.isGenerateHtml()) {
580N/A xrefDir = new File(env.getDataRootFile(), "xref");
580N/A }
580N/A listeners = new ArrayList<IndexChangedListener>();
580N/A dirtyFile = new File(indexDir, "dirty");
580N/A dirty = dirtyFile.exists();
580N/A directories = new ArrayList<String>();
580N/A }
580N/A }
1327N/A
1327N/A /**
580N/A * By default the indexer will traverse all directories in the project.
580N/A * If you add directories with this function update will just process
207N/A * the specified directories.
580N/A *
580N/A * @param dir The directory to scan
1327N/A * @return <code>true</code> if the file is added, false oth
1327N/A */
580N/A @SuppressWarnings("PMD.UseStringBufferForStringAppends")
580N/A public boolean addDirectory(String dir) {
1190N/A String directory = dir;
580N/A if (directory.startsWith("\\")) {
928N/A directory = directory.replace('\\', '/');
207N/A } else if (directory.charAt(0) != '/') {
928N/A directory = "/" + directory;
928N/A }
580N/A File file = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), directory);
1026N/A if (file.exists()) {
580N/A directories.add(directory);
580N/A return true;
580N/A } else {
580N/A return false;
580N/A }
580N/A }
580N/A
580N/A /**
359N/A * Update the content of this index database
207N/A * @throws IOException if an error occurs
207N/A * @throws HistoryException if an error occurs when accessing the history
207N/A */
274N/A public void update() throws IOException, HistoryException {
274N/A synchronized (lock) {
274N/A if (running) {
1190N/A throw new IOException("Indexer already running!");
274N/A }
297N/A running = true;
274N/A interrupted = false;
464N/A }
274N/A
439N/A String ctgs = RuntimeEnvironment.getInstance().getCtags();
439N/A if (ctgs != null) {
439N/A ctags = new Ctags();
464N/A ctags.setBinary(ctgs);
439N/A }
297N/A if (ctags == null) {
439N/A log.severe("Unable to run ctags! searching definitions will not work!");
460N/A }
439N/A
274N/A try {
274N/A //TODO we might need to add writer.commit after certain phases of index generation, right now it will only happen in the end
1185N/A writer = new IndexWriter(indexDirectory, AnalyzerGuru.getAnalyzer(),IndexWriter.MaxFieldLength.UNLIMITED);
274N/A writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
1190N/A
274N/A if (directories.isEmpty()) {
207N/A if (project == null) {
459N/A directories.add("");
678N/A } else {
207N/A directories.add(project.getPath());
1461N/A }
678N/A }
359N/A
359N/A for (String dir : directories) {
1327N/A File sourceRoot;
359N/A if ("".equals(dir)) {
359N/A sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
359N/A } else {
359N/A sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
656N/A }
694N/A
694N/A HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
656N/A
694N/A String startuid = Util.uid(dir, "");
1416N/A IndexReader reader = IndexReader.open(indexDirectory); // open existing index
1416N/A try {
656N/A uidIter = reader.terms(new Term("u", startuid)); // init uid iterator
656N/A
1327N/A indexDown(sourceRoot, dir);
656N/A
656N/A while (uidIter.term() != null && uidIter.term().field().equals("u") && uidIter.term().text().startsWith(startuid)) {
1461N/A removeFile();
207N/A uidIter.next();
1461N/A }
1461N/A } finally {
1461N/A reader.close();
1318N/A }
1318N/A }
1318N/A } finally {
1318N/A if (writer != null) {
1318N/A try {
1318N/A writer.close();
207N/A } catch (IOException e) {
456N/A log.log(Level.WARNING, "An error occured while closing writer", e);
460N/A }
460N/A }
460N/A
274N/A if (ctags != null) {
274N/A try {
207N/A ctags.close();
651N/A } catch (IOException e) {
274N/A log.log(Level.WARNING, "An error occured while closing ctags process", e);
274N/A }
456N/A }
274N/A
274N/A synchronized (lock) {
274N/A running = false;
274N/A }
1190N/A }
651N/A
651N/A if (!isInterrupted() && isDirty()) {
1185N/A if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
1425N/A optimize();
457N/A }
457N/A createSpellingSuggestions();
207N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
1112N/A File timestamp = new File(env.getDataRootFile(), "timestamp");
1108N/A if (timestamp.exists()) {
1114N/A if (!timestamp.setLastModified(System.currentTimeMillis())) {
1327N/A log.warning("Failed to set last modified time on '" + timestamp.getAbsolutePath() + "', used for timestamping the index database.");
1327N/A }
1327N/A } else {
1327N/A if (!timestamp.createNewFile()) {
1327N/A log.warning("Failed to create file '" + timestamp.getAbsolutePath() + "', used for timestamping the index database.");
1327N/A }
1112N/A }
1108N/A }
1108N/A }
207N/A
1327N/A /**
1327N/A * Optimize all index databases
1327N/A * @param executor An executor to run the job
457N/A * @throws IOException if an error occurs
457N/A */
457N/A static void optimizeAll(ExecutorService executor) throws IOException {
457N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
457N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
274N/A if (env.hasProjects()) {
207N/A for (Project project : env.getProjects()) {
207N/A dbs.add(new IndexDatabase(project));
1327N/A }
656N/A } else {
1327N/A dbs.add(new IndexDatabase());
207N/A }
1461N/A
359N/A for (IndexDatabase d : dbs) {
359N/A final IndexDatabase db = d;
359N/A if (db.isDirty()) {
207N/A executor.submit(new Runnable() {
207N/A
359N/A public void run() {
253N/A try {
253N/A db.update();
253N/A } catch (Exception e) {
207N/A log.log(Level.FINE,"Problem updating lucene index database: ",e);
667N/A }
667N/A }
667N/A });
672N/A }
1327N/A }
1327N/A }
1327N/A
672N/A /**
1327N/A * Optimize the index database
1327N/A */
1327N/A public void optimize() {
1327N/A synchronized (lock) {
667N/A if (running) {
207N/A log.warning("Optimize terminated... Someone else is updating / optimizing it!");
207N/A return ;
207N/A }
207N/A running = true;
270N/A }
270N/A IndexWriter wrt = null;
459N/A try {
270N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
312N/A log.info("Optimizing the index ... ");
1190N/A }
270N/A wrt = new IndexWriter(indexDirectory, null, false,IndexWriter.MaxFieldLength.UNLIMITED);
270N/A wrt.optimize();
270N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
564N/A log.info("done");
270N/A }
270N/A synchronized (lock) {
564N/A if (dirtyFile.exists() && !dirtyFile.delete()) {
564N/A log.fine("Failed to remove \"dirty-file\": " +
564N/A dirtyFile.getAbsolutePath());
564N/A }
564N/A dirty = false;
359N/A }
270N/A } catch (IOException e) {
270N/A log.severe("ERROR: optimizing index: " + e);
1054N/A } finally {
270N/A if (wrt != null) {
270N/A try {
270N/A wrt.close();
1264N/A } catch (IOException e) {
1327N/A log.log(Level.WARNING, "An error occured while closing writer", e);
1327N/A }
1327N/A }
270N/A synchronized (lock) {
270N/A running = false;
270N/A }
270N/A }
270N/A }
270N/A
1190N/A /**
270N/A * Generate a spelling suggestion for the definitions stored in defs
207N/A */
207N/A public void createSpellingSuggestions() {
207N/A IndexReader indexReader = null;
359N/A SpellChecker checker = null;
359N/A
1327N/A try {
359N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
359N/A log.info("Generating spelling suggestion index ... ");
359N/A }
359N/A indexReader = IndexReader.open(indexDirectory);
1461N/A checker = new SpellChecker(spellDirectory);
207N/A //TODO below seems only to index "defs" , possible bug ?
1461N/A checker.indexDictionary(new LuceneDictionary(indexReader, "defs"));
1461N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
1190N/A log.info("done");
1327N/A }
1461N/A } catch (IOException e) {
1461N/A log.severe("ERROR: Generating spelling: " + e);
1461N/A } finally {
1318N/A if (indexReader != null) {
1318N/A try {
1318N/A indexReader.close();
1318N/A } catch (IOException e) {
1327N/A log.log(Level.WARNING, "An error occured while closing reader", e);
359N/A }
460N/A }
1327N/A if (spellDirectory != null) {
1327N/A spellDirectory.close();
359N/A }
359N/A }
359N/A }
207N/A
1327N/A private boolean isDirty() {
1327N/A synchronized (lock) {
207N/A return dirty;
1327N/A }
1461N/A }
359N/A
359N/A private void setDirty() {
359N/A synchronized (lock) {
207N/A try {
207N/A if (!dirty && !dirtyFile.createNewFile()) {
207N/A if (!dirtyFile.exists()) {
207N/A log.log(Level.FINE, "Failed to create \"dirty-file\": ", dirtyFile.getAbsolutePath());
207N/A }
207N/A dirty = true;
1461N/A }
207N/A } catch (IOException e) {
207N/A log.log(Level.FINE,"When creating dirty file: ",e);
207N/A }
1461N/A }
1190N/A }
1327N/A /**
1425N/A * Remove a stale file (uidIter.term().text()) from the index database
207N/A * (and the xref file)
830N/A * @throws java.io.IOException if an error occurs
1461N/A */
1461N/A private void removeFile() throws IOException {
1461N/A String path = Util.uid2url(uidIter.term().text());
1318N/A
1461N/A for (IndexChangedListener listener : listeners) {
1461N/A listener.fileRemoved(path);
1327N/A }
207N/A writer.deleteDocuments(uidIter.term());
1461N/A
1461N/A File xrefFile;
1327N/A if (RuntimeEnvironment.getInstance().isCompressXref()) {
207N/A xrefFile = new File(xrefDir, path + ".gz");
1327N/A } else {
1327N/A xrefFile = new File(xrefDir, path);
1461N/A }
207N/A File parent = xrefFile.getParentFile();
207N/A
207N/A if (!xrefFile.delete() && xrefFile.exists()) {
359N/A log.info("Failed to remove obsolete xref-file: " +
359N/A xrefFile.getAbsolutePath());
359N/A }
359N/A
359N/A // Remove the parent directory if it's empty
359N/A if (parent.delete()) {
359N/A log.fine("Removed empty xref dir:" + parent.getAbsolutePath());
359N/A }
359N/A
460N/A setDirty();
1327N/A }
1327N/A
1327N/A /**
359N/A * Add a file to the Lucene index (and generate a xref file)
359N/A * @param file The file to add
359N/A * @param path The path to the file (from source root)
359N/A * @throws java.io.IOException if an error occurs
1327N/A */
253N/A private void addFile(File file, String path) throws IOException {
253N/A final InputStream in =
253N/A new BufferedInputStream(new FileInputStream(file));
207N/A try {
1190N/A FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path);
207N/A fa.setCtags(ctags);
207N/A
207N/A Document d = analyzerGuru.getDocument(file, in, path, fa);
207N/A if (d == null) {
207N/A log.warning("Warning: did not add " + path);
207N/A } else {
207N/A writer.addDocument(d, fa);
1054N/A Genre g = fa.getFactory().getGenre();
207N/A if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
207N/A File xrefFile = new File(xrefDir, path);
207N/A // If mkdirs() returns false, the failure is most likely
1385N/A // because the file already exists. But to check for the
359N/A // file first and only add it if it doesn't exists would
359N/A // only increase the file IO...
602N/A if (!xrefFile.getParentFile().mkdirs()) {
1327N/A assert xrefFile.getParentFile().exists();
1327N/A }
359N/A fa.writeXref(xrefDir, path);
359N/A }
506N/A setDirty();
506N/A for (IndexChangedListener listener : listeners) {
1327N/A listener.fileAdded(path, fa.getClass().getSimpleName());
1327N/A }
506N/A }
253N/A } finally {
1054N/A in.close();
1054N/A }
1190N/A }
207N/A
207N/A /**
207N/A * Check if I should accept this file into the index database
207N/A * @param file the file to check
207N/A * @return true if the file should be included, false otherwise
207N/A */
207N/A private boolean accept(File file) {
207N/A if (ignoredNames.ignore(file)) {
1461N/A return false;
594N/A }
594N/A
594N/A if (!file.canRead()) {
1461N/A log.warning("Warning: could not read " + file.getAbsolutePath());
212N/A return false;
1461N/A }
1054N/A
1054N/A try {
1054N/A if (!file.getAbsolutePath().equals(file.getCanonicalPath())) {
656N/A if (file.getParentFile().equals(file.getCanonicalFile().getParentFile())) {
922N/A // Lets support symlinks within the same directory, this
207N/A // should probably be extended to within the same repository
889N/A return true;
889N/A } else {
889N/A log.warning("Warning: ignored non-local symlink " + file.getAbsolutePath() +
889N/A " -> " + file.getCanonicalPath());
1327N/A return false;
1327N/A }
1327N/A }
1327N/A } catch (IOException exp) {
889N/A log.warning("Warning: Failed to resolve name: " + file.getAbsolutePath());
889N/A log.log(Level.FINE,"Stack Trace: ",exp);
889N/A }
889N/A
889N/A if (file.isDirectory()) {
889N/A // always accept directories so that their files can be examined
889N/A return true;
889N/A }
889N/A
889N/A if (HistoryGuru.getInstance().hasHistory(file)) {
889N/A // versioned files should always be accepted
889N/A return true;
889N/A }
506N/A
889N/A // this is an unversioned file, check if it should be indexed
889N/A return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
889N/A }
889N/A
889N/A /**
207N/A * Generate indexes recursively
553N/A * @param dir the root indexDirectory to generate indexes for
1461N/A * @param path the path
1461N/A */
508N/A private void indexDown(File dir, String parent) throws IOException {
207N/A if (isInterrupted()) {
207N/A return;
207N/A }
207N/A
207N/A if (!accept(dir)) {
207N/A return;
207N/A }
207N/A
1028N/A File[] files = dir.listFiles();
1028N/A if (files == null) {
1028N/A log.severe("Failed to get file listing for: " + dir.getAbsolutePath());
1028N/A return;
1026N/A }
1026N/A Arrays.sort(files, new Comparator<File>() {
207N/A
207N/A public int compare(File p1, File p2) {
207N/A return p1.getName().compareTo(p2.getName());
207N/A }
1016N/A });
1016N/A
207N/A for (File file : files) {
1327N/A if (accept(file)) {
207N/A String path = parent + '/' + file.getName();
207N/A if (file.isDirectory()) {
207N/A indexDown(file, path);
207N/A } else {
1016N/A if (uidIter != null) {
1016N/A String uid = Util.uid(path, DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND)); // construct uid for doc
1327N/A while (uidIter.term() != null && uidIter.term().field().equals("u") &&
1327N/A uidIter.term().text().compareTo(uid) < 0) {
1016N/A removeFile();
207N/A uidIter.next();
876N/A }
872N/A
1327N/A if (uidIter.term() != null && uidIter.term().field().equals("u") &&
1327N/A uidIter.term().text().compareTo(uid) == 0) {
872N/A uidIter.next(); // keep matching docs
207N/A continue;
1327N/A }
1327N/A }
207N/A try {
207N/A addFile(file, path);
504N/A } catch (Exception e) {
504N/A log.log(Level.WARNING,
504N/A "Failed to add file " + file.getAbsolutePath(),
480N/A e);
480N/A }
504N/A }
504N/A }
504N/A }
504N/A }
504N/A
504N/A /**
504N/A * Interrupt the index generation (and the index generation will stop as
207N/A * soon as possible)
207N/A */
1192N/A public void interrupt() {
1192N/A synchronized (lock) {
1192N/A interrupted = true;
1192N/A }
1192N/A }
1327N/A
1327N/A private boolean isInterrupted() {
1192N/A synchronized (lock) {
1192N/A return interrupted;
1192N/A }
1192N/A }
1192N/A
1192N/A /**
1192N/A * Register an object to receive events when modifications is done to the
1327N/A * index database.
1327N/A *
1192N/A * @param listener the object to receive the events
1192N/A */
1192N/A public void addIndexChangedListener(IndexChangedListener listener) {
1192N/A listeners.add(listener);
1192N/A }
1192N/A
1327N/A /**
1327N/A * Remove an object from the lists of objects to receive events when
1192N/A * modifications is done to the index database
1192N/A *
1192N/A * @param listener the object to remove
1192N/A */
207N/A public void removeIndexChangedListener(IndexChangedListener listener) {
1016N/A listeners.remove(listener);
1016N/A }
1016N/A
1016N/A /**
1016N/A * List all files in all of the index databases
1016N/A * @throws IOException if an error occurs
1051N/A */
1051N/A public static void listAllFiles() throws IOException {
1051N/A listAllFiles(null);
1051N/A }
1051N/A
1016N/A /**
1016N/A * List all files in some of the index databases
1016N/A * @param subFiles Subdirectories for the various projects to list the files
1016N/A * for (or null or an empty list to dump all projects)
1016N/A * @throws IOException if an error occurs
1016N/A */
1016N/A public static void listAllFiles(List<String> subFiles) throws IOException {
1016N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
1016N/A if (env.hasProjects()) {
1016N/A if (subFiles == null || subFiles.isEmpty()) {
1016N/A for (Project project : env.getProjects()) {
1016N/A IndexDatabase db = new IndexDatabase(project);
1016N/A db.listFiles();
1051N/A }
1051N/A } else {
1051N/A for (String path : subFiles) {
1051N/A Project project = Project.getProject(path);
1051N/A if (project == null) {
1051N/A log.warning("Warning: Could not find a project for \"" + path + "\"");
1051N/A } else {
1051N/A IndexDatabase db = new IndexDatabase(project);
1051N/A db.listFiles();
1051N/A }
1051N/A }
1051N/A }
1051N/A } else {
1051N/A IndexDatabase db = new IndexDatabase();
1051N/A db.listFiles();
1051N/A }
1051N/A }
1051N/A
1051N/A /**
1051N/A * List all of the files in this index database
1051N/A *
1051N/A * @throws IOException If an IO error occurs while reading from the database
1051N/A */
1051N/A public void listFiles() throws IOException {
1051N/A IndexReader ireader = null;
1051N/A TermEnum iter = null;
1051N/A
1051N/A try {
1051N/A ireader = IndexReader.open(indexDirectory); // open existing index
1051N/A iter = ireader.terms(new Term("u", "")); // init uid iterator
207N/A while (iter.term() != null) {
207N/A log.info(Util.uid2url(iter.term().text()));
207N/A iter.next();
1112N/A }
1112N/A } finally {
1112N/A if (iter != null) {
1112N/A try {
207N/A iter.close();
1461N/A } catch (IOException e) {
1108N/A log.log(Level.WARNING, "An error occured while closing index iterator", e);
1112N/A }
359N/A }
1112N/A
207N/A if (ireader != null) {
207N/A try {
207N/A ireader.close();
1112N/A } catch (IOException e) {
207N/A log.log(Level.WARNING, "An error occured while closing index reader", e);
207N/A }
207N/A }
207N/A }
1327N/A }
1112N/A
207N/A static void listFrequentTokens() throws IOException {
282N/A listFrequentTokens(null);
1054N/A }
282N/A
282N/A static void listFrequentTokens(List<String> subFiles) throws IOException {
282N/A final int limit = 4;
282N/A
207N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
207N/A if (env.hasProjects()) {
1192N/A if (subFiles == null || subFiles.isEmpty()) {
207N/A for (Project project : env.getProjects()) {
1108N/A IndexDatabase db = new IndexDatabase(project);
207N/A db.listTokens(4);
1112N/A }
207N/A } else {
1112N/A for (String path : subFiles) {
1112N/A Project project = Project.getProject(path);
1108N/A if (project == null) {
1112N/A log.warning("Warning: Could not find a project for \"" + path + "\"");
1108N/A } else {
1327N/A IndexDatabase db = new IndexDatabase(project);
1327N/A db.listTokens(4);
1190N/A }
1327N/A }
1327N/A }
1327N/A } else {
1108N/A IndexDatabase db = new IndexDatabase();
1108N/A db.listTokens(limit);
594N/A }
1327N/A }
1327N/A
1327N/A public void listTokens(int freq) throws IOException {
207N/A IndexReader ireader = null;
207N/A TermEnum iter = null;
207N/A
207N/A try {
207N/A ireader = IndexReader.open(indexDirectory);
207N/A iter = ireader.terms(new Term("defs", ""));
207N/A while (iter.term() != null) {
207N/A if (iter.term().field().startsWith("f")) {
972N/A if (iter.docFreq() > 16 && iter.term().text().length() > freq) {
594N/A log.warning(iter.term().text());
207N/A }
207N/A iter.next();
594N/A } else {
594N/A break;
594N/A }
1327N/A }
1327N/A } finally {
594N/A if (iter != null) {
207N/A try {
207N/A iter.close();
207N/A } catch (IOException e) {
1108N/A log.log(Level.WARNING, "An error occured while closing index iterator", e);
1112N/A }
207N/A }
207N/A
207N/A if (ireader != null) {
207N/A try {
207N/A ireader.close();
207N/A } catch (IOException e) {
207N/A log.log(Level.WARNING, "An error occured while closing index reader", e);
359N/A }
359N/A }
359N/A }
359N/A }
359N/A
359N/A /**
359N/A * Get an indexReader for the Index database where a given file
359N/A * @param path the file to get the database for
1190N/A * @return The index database where the file should be located or null if
207N/A * it cannot be located.
207N/A */
207N/A public static IndexReader getIndexReader(String path) {
207N/A IndexReader ret = null;
207N/A
1190N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
207N/A File indexDir = new File(env.getDataRootFile(), "index");
207N/A
316N/A if (env.hasProjects()) {
207N/A Project p = Project.getProject(path);
207N/A if (p == null) {
207N/A return null;
207N/A } else {
207N/A indexDir = new File(indexDir, p.getPath());
207N/A }
1190N/A }
207N/A
207N/A if (indexDir.exists() && IndexReader.indexExists(indexDir)) {
316N/A try {
207N/A ret = IndexReader.open(indexDir);
207N/A } catch (Exception ex) {
207N/A log.severe("Failed to open index: " + indexDir.getAbsolutePath());
207N/A log.log(Level.FINE,"Stack Trace: ",ex);
207N/A }
359N/A }
207N/A
312N/A return ret;
207N/A }
207N/A
207N/A @Override
207N/A public boolean equals(Object obj) {
207N/A if (obj == null) {
207N/A return false;
207N/A }
359N/A if (getClass() != obj.getClass()) {
207N/A return false;
312N/A }
207N/A final IndexDatabase other = (IndexDatabase) obj;
460N/A if (this.project != other.project && (this.project == null || !this.project.equals(other.project))) {
207N/A return false;
207N/A }
207N/A return true;
207N/A }
207N/A
207N/A @Override
207N/A public int hashCode() {
207N/A int hash = 7;
207N/A hash = 41 * hash + (this.project == null ? 0 : this.project.hashCode());
1327N/A return hash;
1327N/A }
207N/A
207N/A}
207N/A