IndexDatabase.java revision 1463
207N/A/*
207N/A * CDDL HEADER START
207N/A *
207N/A * The contents of this file are subject to the terms of the
207N/A * Common Development and Distribution License (the "License").
207N/A * You may not use this file except in compliance with the License.
207N/A *
207N/A * See LICENSE.txt included in this distribution for the specific
207N/A * language governing permissions and limitations under the License.
207N/A *
207N/A * When distributing Covered Code, include this CDDL HEADER in each
207N/A * file and include the License file at LICENSE.txt.
207N/A * If applicable, add the following below this CDDL HEADER, with the
207N/A * fields enclosed by brackets "[]" replaced with your own identifying
207N/A * information: Portions Copyright [yyyy] [name of copyright owner]
207N/A *
207N/A * CDDL HEADER END
207N/A */
207N/A
207N/A/*
207N/A * Copyright (c) 2008, 2012, Oracle and/or its affiliates. All rights reserved.
207N/A */
207N/A
207N/Apackage org.opensolaris.opengrok.index;
207N/A
207N/Aimport java.io.BufferedInputStream;
207N/Aimport java.io.File;
207N/Aimport java.io.FileInputStream;
207N/Aimport java.io.FileNotFoundException;
207N/Aimport java.io.IOException;
207N/Aimport java.io.InputStream;
207N/Aimport java.util.ArrayList;
207N/Aimport java.util.Arrays;
282N/Aimport java.util.Comparator;
207N/Aimport java.util.List;
261N/Aimport java.util.concurrent.ExecutorService;
320N/Aimport java.util.logging.Level;
312N/Aimport java.util.logging.Logger;
207N/A
207N/Aimport org.apache.lucene.analysis.Analyzer;
207N/Aimport org.apache.lucene.analysis.standard.StandardAnalyzer;
207N/Aimport org.apache.lucene.document.DateTools;
207N/Aimport org.apache.lucene.document.Document;
207N/Aimport org.apache.lucene.document.Fieldable;
207N/Aimport org.apache.lucene.index.IndexReader;
207N/Aimport org.apache.lucene.index.IndexWriter;
207N/Aimport org.apache.lucene.index.IndexWriterConfig;
207N/Aimport org.apache.lucene.index.IndexWriterConfig.OpenMode;
656N/Aimport org.apache.lucene.index.Term;
207N/Aimport org.apache.lucene.index.TermEnum;
207N/Aimport org.apache.lucene.queryParser.ParseException;
207N/Aimport org.apache.lucene.search.IndexSearcher;
207N/Aimport org.apache.lucene.search.Query;
678N/Aimport org.apache.lucene.search.TopDocs;
480N/Aimport org.apache.lucene.search.spell.LuceneDictionary;
207N/Aimport org.apache.lucene.search.spell.SpellChecker;
207N/Aimport org.apache.lucene.store.FSDirectory;
207N/Aimport org.apache.lucene.store.LockFactory;
207N/Aimport org.apache.lucene.store.NoLockFactory;
207N/Aimport org.apache.lucene.store.SimpleFSLockFactory;
207N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
207N/Aimport org.opensolaris.opengrok.analysis.Ctags;
207N/Aimport org.opensolaris.opengrok.analysis.Definitions;
207N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer;
207N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
207N/Aimport org.opensolaris.opengrok.configuration.Project;
207N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
207N/Aimport org.opensolaris.opengrok.history.HistoryException;
207N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
207N/Aimport org.opensolaris.opengrok.search.QueryBuilder;
207N/Aimport org.opensolaris.opengrok.search.SearchEngine;
207N/Aimport org.opensolaris.opengrok.util.IOUtils;
207N/Aimport org.opensolaris.opengrok.web.Util;
207N/A
207N/A/**
253N/A * This class is used to create / update the index databases. Currently we use
359N/A * one index database per project.
207N/A *
359N/A * @author Trond Norbye
274N/A * @author Lubos Kosco , update for lucene 3.0.0
320N/A */
656N/Apublic class IndexDatabase {
656N/A
207N/A private Project project;
207N/A private FSDirectory indexDirectory;
207N/A private FSDirectory spellDirectory;
207N/A private IndexWriter writer;
207N/A private TermEnum uidIter;
207N/A private IgnoredNames ignoredNames;
207N/A private Filter includedNames;
207N/A private AnalyzerGuru analyzerGuru;
207N/A private File xrefDir;
207N/A private boolean interrupted;
207N/A private List<IndexChangedListener> listeners;
207N/A private File dirtyFile;
207N/A private final Object lock = new Object();
207N/A private boolean dirty;
207N/A private boolean running;
207N/A private List<String> directories;
207N/A static final Logger logger = Logger.getLogger(IndexDatabase.class.getName());
207N/A private Ctags ctags;
207N/A private LockFactory lockfact;
207N/A
207N/A /**
207N/A * Create a new instance of the Index Database. Use this constructor if
207N/A * you don't use any projects
261N/A *
459N/A * @throws java.io.IOException if an error occurs while creating directories
207N/A */
459N/A public IndexDatabase() throws IOException {
261N/A this(null);
207N/A }
207N/A
207N/A /**
207N/A * Create a new instance of an Index Database for a given project
261N/A * @param project the project to create the database for
207N/A * @throws java.io.IOException if an errror occurs while creating directories
459N/A */
207N/A public IndexDatabase(Project project) throws IOException {
312N/A this.project = project;
207N/A lockfact = new SimpleFSLockFactory();
564N/A initialize();
564N/A }
207N/A
207N/A /**
564N/A * Update the index database for all of the projects. Print progress to
207N/A * standard out.
207N/A * @param executor An executor to run the job
564N/A * @throws IOException if an error occurs
564N/A */
564N/A public static void updateAll(ExecutorService executor) throws IOException {
564N/A updateAll(executor, null);
564N/A }
207N/A
207N/A /**
207N/A * Update the index database for all of the projects
261N/A * @param executor An executor to run the job
261N/A * @param listener where to signal the changes to the database
261N/A * @throws IOException if an error occurs
261N/A */
261N/A static void updateAll(ExecutorService executor, IndexChangedListener listener) throws IOException {
261N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
261N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
320N/A
261N/A if (env.hasProjects()) {
261N/A for (Project project : env.getProjects()) {
261N/A dbs.add(new IndexDatabase(project));
207N/A }
207N/A } else {
207N/A dbs.add(new IndexDatabase());
668N/A }
668N/A
668N/A for (IndexDatabase d : dbs) {
668N/A final IndexDatabase db = d;
668N/A if (listener != null) {
668N/A db.addIndexChangedListener(listener);
668N/A }
668N/A
668N/A executor.submit(new Runnable() {
668N/A
668N/A @Override
668N/A public void run() {
668N/A try {
668N/A db.update();
668N/A } catch (Throwable e) {
668N/A logger.warning("Problem updating " + db + ": "+ e.getMessage());
668N/A logger.log(Level.FINE, "updateAll", e);
668N/A }
668N/A }
668N/A });
668N/A }
668N/A }
668N/A
668N/A /**
668N/A * Update the index database for a number of sub-directories
668N/A * @param executor An executor to run the job
668N/A * @param listener where to signal the changes to the database
668N/A * @param paths
668N/A */
668N/A public static void update(ExecutorService executor, IndexChangedListener listener, List<String> paths) {
668N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
668N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
668N/A
668N/A for (String path : paths) {
668N/A Project project = Project.getProject(path);
668N/A if (project == null && env.hasProjects()) {
668N/A logger.warning("Could not find a project for '" + path + "'");
668N/A } else {
668N/A IndexDatabase db = null;
668N/A
668N/A try {
668N/A if (project == null) {
668N/A db = new IndexDatabase();
668N/A } else {
668N/A db = new IndexDatabase(project);
668N/A }
668N/A
668N/A int idx = dbs.indexOf(db);
668N/A if (idx != -1) {
668N/A db = dbs.get(idx);
668N/A }
668N/A
668N/A if (db.addDirectory(path)) {
668N/A if (idx == -1) {
668N/A dbs.add(db);
668N/A }
668N/A } else {
668N/A logger.warning("Directory does not exist '" + path + "'");
668N/A }
460N/A } catch (IOException e) {
580N/A logger.warning("An error occured while updating "
580N/A + db + ": " + e.getMessage());
580N/A logger.log(Level.FINE, "update", e);
580N/A }
580N/A }
580N/A
580N/A for (final IndexDatabase db : dbs) {
580N/A db.addIndexChangedListener(listener);
580N/A executor.submit(new Runnable() {
580N/A
580N/A @Override
580N/A public void run() {
580N/A try {
580N/A db.update();
580N/A } catch (Throwable e) {
580N/A logger.warning("An error occured while updating "
207N/A + db + ": " + e.getLocalizedMessage());
580N/A logger.log(Level.FINE, "run", e);
580N/A }
580N/A }
580N/A });
580N/A }
580N/A }
580N/A }
580N/A
207N/A @SuppressWarnings("PMD.CollapsibleIfStatements")
580N/A private void initialize() throws IOException {
580N/A synchronized (this) {
580N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
580N/A File indexDir = new File(env.getDataRootFile(), "index");
580N/A File spellDir = new File(env.getDataRootFile(), "spellIndex");
580N/A if (project != null) {
580N/A indexDir = new File(indexDir, project.getPath());
580N/A spellDir = new File(spellDir, project.getPath());
580N/A }
580N/A
580N/A if (!indexDir.exists() && !indexDir.mkdirs()) {
359N/A // to avoid race conditions, just recheck..
207N/A if (!indexDir.exists()) {
207N/A throw new FileNotFoundException("Failed to create root directory '"
207N/A + indexDir.getAbsolutePath() + "'");
274N/A }
274N/A }
274N/A
274N/A if (!spellDir.exists() && !spellDir.mkdirs()) {
274N/A if (!spellDir.exists()) {
297N/A throw new FileNotFoundException("Failed to create root directory '"
274N/A + spellDir.getAbsolutePath() + "'");
464N/A }
274N/A }
439N/A
439N/A if (!env.isUsingLuceneLocking()) {
439N/A lockfact = NoLockFactory.getNoLockFactory();
464N/A }
439N/A indexDirectory = FSDirectory.open(indexDir,lockfact);
297N/A spellDirectory = FSDirectory.open(spellDir,lockfact);
439N/A ignoredNames = env.getIgnoredNames();
460N/A includedNames = env.getIncludedNames();
439N/A analyzerGuru = new AnalyzerGuru();
274N/A if (env.isGenerateHtml()) {
460N/A xrefDir = new File(env.getDataRootFile(), "xref");
460N/A }
274N/A listeners = new ArrayList<IndexChangedListener>();
274N/A dirtyFile = new File(indexDir, "dirty");
274N/A dirty = dirtyFile.exists();
274N/A directories = new ArrayList<String>();
207N/A }
459N/A }
678N/A
207N/A /**
678N/A * By default the indexer will traverse all directories in the project.
359N/A * If you add directories with this function update will just process
359N/A * the specified directories.
459N/A *
359N/A * @param dir The directory to scan
359N/A * @return <code>true</code> if the file is added, false oth
359N/A */
359N/A @SuppressWarnings("PMD.UseStringBufferForStringAppends")
656N/A public boolean addDirectory(String dir) {
694N/A String directory = dir;
694N/A if (directory.startsWith("\\")) {
656N/A directory = directory.replace('\\', '/');
694N/A } else if (directory.charAt(0) != '/') {
656N/A directory = "/" + directory;
656N/A }
656N/A File file = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), directory);
656N/A if (file.exists()) {
656N/A directories.add(directory);
207N/A return true;
816N/A }
816N/A return false;
255N/A }
207N/A
456N/A /**
460N/A * Update the content of this index database
460N/A * @throws IOException if an error occurs
460N/A * @throws HistoryException if an error occurs when accessing the history
274N/A */
274N/A @SuppressWarnings({ "resource", "boxing" })
207N/A public void update() throws IOException, HistoryException {
651N/A synchronized (lock) {
274N/A if (running) {
274N/A throw new IOException("Indexer already running");
456N/A }
274N/A running = true;
274N/A interrupted = false;
274N/A }
274N/A
274N/A String ctgs = RuntimeEnvironment.getInstance().getCtags();
651N/A if (ctgs != null) {
651N/A ctags = new Ctags();
274N/A ctags.setBinary(ctgs);
457N/A ctags.setOptionsFile(RuntimeEnvironment.getInstance()
457N/A .getCtagsOptionsFile());
457N/A }
207N/A if (ctags == null) {
457N/A logger.warning("Unable to run ctags! Searching definitions will not work!");
207N/A }
457N/A
457N/A Analyzer analyzer = null;
457N/A try {
457N/A //TODO we might need to add writer.commit after certain phases of
457N/A // index generation, right now it will only happen in the end
457N/A analyzer = AnalyzerGuru.getAnalyzer();
274N/A IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
207N/A iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
207N/A //iwc.setRAMBufferSizeMB(256.0); //TODO check what is the sweet spot
207N/A writer = new IndexWriter(indexDirectory, iwc);
816N/A writer.commit(); // to make sure index exists on the disk
207N/A //writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
207N/A
508N/A if (directories.isEmpty()) {
207N/A if (project == null) {
207N/A directories.add("");
656N/A } else {
656N/A directories.add(project.getPath());
656N/A }
656N/A }
656N/A
656N/A for (String dir : directories) {
656N/A File sourceRoot;
656N/A if ("".equals(dir)) {
656N/A sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
359N/A } else {
359N/A sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
359N/A }
207N/A
207N/A HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
359N/A
253N/A String startuid = Util.path2uid(dir, "");
253N/A IndexReader reader = IndexReader.open(indexDirectory); // open existing index
253N/A try {
207N/A uidIter = reader.terms(new Term("u", startuid)); // init uid iterator
667N/A
667N/A //TODO below should be optional, since it traverses the tree once more to get total count! :(
667N/A int file_cnt = 0;
672N/A if (RuntimeEnvironment.getInstance().isPrintProgress()) {
672N/A logger.log(Level.INFO, "Counting files in ''{0}'' ...", dir);
672N/A file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
667N/A if (logger.isLoggable(Level.INFO)) {
672N/A logger.log(Level.INFO, "Need to process {0} files for ''{1}''",
672N/A new Object[] { file_cnt, dir });
672N/A }
667N/A }
207N/A
207N/A indexDown(sourceRoot, dir, false, 0, file_cnt);
207N/A
207N/A while (uidIter.term() != null && uidIter.term().field().equals("u")
270N/A && uidIter.term().text().startsWith(startuid))
270N/A {
459N/A removeFile();
270N/A uidIter.next();
312N/A }
564N/A } finally {
270N/A reader.close();
270N/A }
270N/A }
564N/A } finally {
270N/A IOUtils.close(writer);
270N/A if (ctags != null) {
564N/A ctags.close();
564N/A }
564N/A IOUtils.close(analyzer);
564N/A synchronized (lock) {
564N/A running = false;
359N/A }
270N/A }
270N/A
270N/A if (!isInterrupted() && isDirty()) {
270N/A if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
270N/A optimize();
678N/A }
320N/A createSpellingSuggestions();
270N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
270N/A File timestamp = new File(env.getDataRootFile(), "timestamp");
270N/A if (timestamp.exists()) {
270N/A if (!timestamp.setLastModified(System.currentTimeMillis())) {
270N/A logger.warning("Failed to set last modified time on '"
270N/A + timestamp.getAbsolutePath()
270N/A + "'used for timestamping the index database");
270N/A }
207N/A } else if (!timestamp.createNewFile()) {
207N/A logger.warning("Failed to create file '"
207N/A + timestamp.getAbsolutePath()
359N/A + "', used for timestamping the index database");
359N/A }
359N/A }
359N/A }
359N/A
359N/A /**
359N/A * Optimize all index databases
207N/A * @param executor An executor to run the job
207N/A * @throws IOException if an error occurs
207N/A */
320N/A static void optimizeAll(ExecutorService executor) throws IOException {
207N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
816N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
207N/A if (env.hasProjects()) {
207N/A for (Project project : env.getProjects()) {
320N/A dbs.add(new IndexDatabase(project));
207N/A }
359N/A } else {
460N/A dbs.add(new IndexDatabase());
460N/A }
460N/A
359N/A for (IndexDatabase d : dbs) {
359N/A final IndexDatabase db = d;
359N/A if (db.isDirty()) {
207N/A executor.submit(new Runnable() {
320N/A
207N/A @Override
207N/A public void run() {
207N/A try {
207N/A db.update();
207N/A } catch (Throwable e) {
508N/A logger.warning("Problem updating " + db + ": "
207N/A + e.getMessage());
207N/A logger.log(Level.FINE, "optimizeAll", e);
359N/A }
359N/A }
359N/A });
207N/A }
207N/A }
207N/A }
207N/A
207N/A /**
207N/A * Optimize the index database
207N/A */
207N/A public void optimize() {
207N/A synchronized (lock) {
207N/A if (running) {
207N/A logger.warning("Optimize terminated... Someone else is updating / optimizing it!");
207N/A return ;
320N/A }
207N/A running = true;
207N/A }
207N/A @SuppressWarnings("resource")
830N/A IndexWriter wrt = null;
207N/A @SuppressWarnings("resource")
207N/A Analyzer analyzer = null;
320N/A try {
207N/A logger.info("Optimizing " + this + " ...");
207N/A analyzer = new StandardAnalyzer(SearchEngine.LUCENE_VERSION);
320N/A IndexWriterConfig conf =
207N/A new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
207N/A conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
207N/A
207N/A wrt = new IndexWriter(indexDirectory, conf);
207N/A wrt.forceMerge(1); // this is deprecated and not needed anymore
508N/A logger.info("Optimizing " + this + " done");
207N/A synchronized (lock) {
207N/A if (dirtyFile.exists() && !dirtyFile.delete()) {
207N/A logger.log(Level.FINE, "Failed to remove \"dirty-file\" ''{0}''",
207N/A dirtyFile.getAbsolutePath());
207N/A }
207N/A dirty = false;
207N/A }
207N/A } catch (IOException e) {
359N/A logger.warning(this + " optimizing problem: " + e.getMessage());
359N/A logger.log(Level.FINE, "optimize", e);
359N/A } finally {
359N/A IOUtils.close(wrt);
359N/A IOUtils.close(analyzer);
359N/A synchronized (lock) {
359N/A running = false;
359N/A }
359N/A }
460N/A }
460N/A
460N/A /**
359N/A * Generate a spelling suggestion for the definitions stored in defs
359N/A */
359N/A @SuppressWarnings("resource")
359N/A public void createSpellingSuggestions() {
359N/A IndexReader indexReader = null;
253N/A SpellChecker checker = null;
253N/A Analyzer analyzer = null;
253N/A try {
207N/A logger.info("Generating spelling suggestions for " + this + " ...");
207N/A indexReader = IndexReader.open(indexDirectory);
207N/A checker = new SpellChecker(spellDirectory);
207N/A //TODO below seems only to index "defs" , possible bug ?
207N/A analyzer = AnalyzerGuru.getAnalyzer();
207N/A IndexWriterConfig iwc =
207N/A new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
207N/A iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
207N/A checker.indexDictionary(new LuceneDictionary(indexReader, "defs"),
207N/A iwc, false);
207N/A logger.info("Generating spelling suggestions for " + this + " done");
207N/A } catch (IOException e) {
207N/A logger.warning("Problem when generating spelling suggestions for "
515N/A + this + ": " + e.getMessage());
515N/A logger.log(Level.FINE, "createSpellingSuggestions", e);
515N/A } finally {
515N/A IOUtils.close(indexReader);
515N/A IOUtils.close(spellDirectory);
515N/A IOUtils.close(analyzer);
359N/A }
359N/A }
602N/A
359N/A private boolean isDirty() {
359N/A synchronized (lock) {
359N/A return dirty;
359N/A }
506N/A }
506N/A
506N/A private void setDirty() {
506N/A synchronized (lock) {
359N/A try {
253N/A if (!dirty && !dirtyFile.createNewFile()) {
207N/A if (!dirtyFile.exists() && logger.isLoggable(Level.FINE)) {
207N/A logger.fine("Failed to create \"dirty-file\" '" +
207N/A dirtyFile.getAbsolutePath() + "'");
207N/A }
207N/A dirty = true;
207N/A }
207N/A } catch (IOException e) {
207N/A logger.log(Level.FINE,"unable to creating dirty file", e);
594N/A }
594N/A }
594N/A }
212N/A /**
553N/A * Remove a stale file (uidIter.term().text()) from the index database
656N/A * (and the xref file)
207N/A * @throws java.io.IOException if an error occurs
553N/A */
553N/A private void removeFile() throws IOException {
553N/A String path = Util.uid2url(uidIter.term().text());
553N/A
553N/A for (IndexChangedListener listener : listeners) {
553N/A listener.fileRemove(path);
553N/A }
553N/A writer.deleteDocuments(uidIter.term());
553N/A
553N/A File xrefFile = new File(xrefDir, path);
553N/A File parent = xrefFile.getParentFile();
553N/A
553N/A if (!xrefFile.delete() && xrefFile.exists()) {
553N/A logger.log(Level.INFO, "Failed to remove obsolete xref-file ''{0}''",
553N/A xrefFile.getAbsolutePath());
553N/A }
506N/A
553N/A // Remove the parent directory if it's empty
593N/A if (parent.delete()) {
593N/A logger.log(Level.FINE, "Removed empty xref dir ''{0}''",
593N/A parent.getAbsolutePath());
207N/A }
553N/A setDirty();
594N/A for (IndexChangedListener listener : listeners) {
508N/A listener.fileRemoved(path);
207N/A }
207N/A }
207N/A
207N/A /**
207N/A * Add a file to the Lucene index (and generate a xref file)
207N/A * @param file The file to add
207N/A * @param path The path to the file (from source root)
207N/A * @throws java.io.IOException if an error occurs
207N/A */
207N/A @SuppressWarnings("resource")
207N/A private void addFile(File file, String path) throws IOException {
207N/A final InputStream in =
207N/A new BufferedInputStream(new FileInputStream(file));
320N/A FileAnalyzer fa = null;
207N/A try {
207N/A fa = AnalyzerGuru.getAnalyzer(in, path);
207N/A for (IndexChangedListener listener : listeners) {
207N/A listener.fileAdd(path, fa.getClass().getSimpleName());
207N/A }
310N/A fa.setCtags(ctags);
310N/A fa.setProject(Project.getProject(path));
310N/A
310N/A Document d;
310N/A try {
320N/A d = analyzerGuru.getDocument(file, in, path, fa);
310N/A } catch (Exception e) {
310N/A logger.log(Level.INFO,
310N/A "Skipped file ''{0}'' because the analyzer didn''t " +
207N/A "understand it.", path);
207N/A logger.log(Level.FINE, "addFile", e);
320N/A return;
320N/A }
207N/A
207N/A writer.addDocument(d, fa);
504N/A Genre g = fa.getFactory().getGenre();
504N/A if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
504N/A File xrefFile = new File(xrefDir, path);
480N/A // If mkdirs() returns false, the failure is most likely
480N/A // because the file already exists. But to check for the
504N/A // file first and only add it if it doesn't exists would
504N/A // only increase the file IO...
504N/A if (!xrefFile.getParentFile().mkdirs()) {
504N/A assert xrefFile.getParentFile().exists();
504N/A }
504N/A fa.writeXref(xrefDir, path);
504N/A }
207N/A setDirty();
207N/A for (IndexChangedListener listener : listeners) {
207N/A listener.fileAdded(path, fa.getClass().getSimpleName());
207N/A }
207N/A } finally {
207N/A IOUtils.close(in);
207N/A IOUtils.close(fa);
207N/A }
359N/A }
207N/A
207N/A /**
207N/A * Check if I should accept this file into the index database
207N/A * @param file the file to check
207N/A * @return true if the file should be included, false otherwise
207N/A */
207N/A private boolean accept(File file) {
207N/A
207N/A if (!includedNames.isEmpty() &&
320N/A // the filter should not affect directory names
207N/A (!(file.isDirectory() || includedNames.match(file))) ) {
207N/A return false;
282N/A }
282N/A if (ignoredNames.ignore(file)) {
282N/A return false;
282N/A }
282N/A
282N/A String absolutePath = file.getAbsolutePath();
207N/A
207N/A if (!file.canRead()) {
207N/A logger.warning("Could not read " + absolutePath);
207N/A return false;
207N/A }
207N/A
207N/A try {
594N/A String canonicalPath = file.getCanonicalPath();
207N/A if (!absolutePath.equals(canonicalPath) && !acceptSymlink(absolutePath, canonicalPath)) {
207N/A logger.log(Level.FINE, "Skipped symlink ''{0}'' -> ''{1}''",
207N/A new Object[]{absolutePath, canonicalPath});
207N/A return false;
207N/A }
207N/A //below will only let go files and directories, anything else is considered special and is not added
207N/A if (!file.isFile() && !file.isDirectory()) {
207N/A logger.warning("Ignored special file '" + absolutePath + "'");
207N/A return false;
207N/A }
594N/A } catch (IOException exp) {
207N/A logger.warning("Failed to resolve name '" + absolutePath + "'");
207N/A logger.log(Level.FINE, "accept", exp);
594N/A }
594N/A
594N/A if (file.isDirectory()) {
594N/A // always accept directories so that their files can be examined
594N/A return true;
594N/A }
594N/A
207N/A if (HistoryGuru.getInstance().hasHistory(file)) {
207N/A // versioned files should always be accepted
207N/A return true;
207N/A }
207N/A
207N/A // this is an unversioned file, check if it should be indexed
207N/A return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
207N/A }
207N/A
207N/A boolean accept(File parent, File file) {
359N/A try {
359N/A File f1 = parent.getCanonicalFile();
359N/A File f2 = file.getCanonicalFile();
359N/A if (f1.equals(f2)) {
359N/A logger.log(Level.INFO, "Skipping links to itself (''{0}'' ''{1}'')",
359N/A new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
359N/A return false;
359N/A }
359N/A
207N/A // Now, let's verify that it's not a link back up the chain...
207N/A File t1 = f1;
207N/A while ((t1 = t1.getParentFile()) != null) {
207N/A if (f2.equals(t1)) {
207N/A logger.log(Level.INFO, "Skipping links to parent (''{0}'' ''{1}'')",
207N/A new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
207N/A return false;
207N/A }
316N/A }
207N/A
207N/A return accept(file);
207N/A } catch (IOException ex) {
207N/A logger.log(Level.WARNING, "Failed to resolve name (''{0}'' ''{1}'')",
207N/A new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
207N/A }
207N/A return false;
207N/A }
207N/A
316N/A /**
207N/A * Check if I should accept the path containing a symlink
207N/A * @param absolutePath the path with a symlink to check
207N/A * @param canonicalPath the canonical path to the file
207N/A * @return true if the file should be accepted, false otherwise
207N/A */
359N/A private boolean acceptSymlink(String absolutePath, String canonicalPath) throws IOException {
207N/A // Always accept local symlinks
312N/A if (isLocal(canonicalPath)) {
207N/A return true;
207N/A }
207N/A
207N/A for (String allowedSymlink : RuntimeEnvironment.getInstance().getAllowedSymlinks()) {
207N/A if (absolutePath.startsWith(allowedSymlink)) {
207N/A String allowedTarget = new File(allowedSymlink).getCanonicalPath();
207N/A if (canonicalPath.startsWith(allowedTarget) &&
359N/A absolutePath.substring(allowedSymlink.length()).equals(canonicalPath.substring(allowedTarget.length()))) {
207N/A return true;
312N/A }
207N/A }
460N/A }
207N/A return false;
207N/A }
207N/A
207N/A /**
207N/A * Check if a file is local to the current project. If we don't have
207N/A * projects, check if the file is in the source root.
207N/A *
207N/A * @param path the path to a file
207N/A * @return true if the file is local to the current repository
320N/A */
207N/A private boolean isLocal(String path) {
207N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
207N/A String srcRoot = env.getSourceRootPath();
207N/A
207N/A boolean local = false;
207N/A
460N/A if (path.startsWith(srcRoot)) {
460N/A if (env.hasProjects()) {
460N/A String relPath = path.substring(srcRoot.length());
207N/A if (project.equals(Project.getProject(relPath))) {
207N/A // File is under the current project, so it's local.
207N/A local = true;
207N/A }
207N/A } else {
207N/A // File is under source root, and we don't have projects, so
359N/A // consider it local.
207N/A local = true;
312N/A }
207N/A }
207N/A
207N/A return local;
207N/A }
207N/A
207N/A /**
207N/A * Generate indexes recursively
320N/A * @param dir the root indexDirectory to generate indexes for
207N/A * @param path the path
207N/A * @param count_only if true will just traverse the source root and count files
207N/A * @param cur_count current count during the traversal of the tree
207N/A * @param est_total estimate total files to process
207N/A *
207N/A */
459N/A @SuppressWarnings("boxing")
580N/A private int indexDown(File dir, String parent, boolean count_only, int cur_count, int est_total) throws IOException {
207N/A int lcur_count=cur_count;
207N/A if (isInterrupted()) {
207N/A return lcur_count;
207N/A }
207N/A
207N/A if (!accept(dir)) {
459N/A return lcur_count;
508N/A }
207N/A
207N/A File[] files = dir.listFiles();
207N/A if (files == null) {
207N/A logger.severe("Failed to get file listing for '" + dir.getAbsolutePath() + "'");
207N/A return lcur_count;
459N/A }
207N/A Arrays.sort(files, new Comparator<File>() {
207N/A @Override
207N/A public int compare(File p1, File p2) {
457N/A return p1.getName().compareTo(p2.getName());
207N/A }
207N/A });
207N/A
460N/A for (File file : files) {
207N/A if (accept(dir, file)) {
207N/A String path = parent + '/' + file.getName();
207N/A
207N/A if (file.isDirectory()) {
207N/A lcur_count = indexDown(file, path, count_only, lcur_count, est_total);
207N/A } else {
207N/A lcur_count++;
207N/A if (count_only) {
207N/A continue;
320N/A }
207N/A
207N/A if (RuntimeEnvironment.getInstance().isPrintProgress()
207N/A && est_total > 0 && logger.isLoggable(Level.INFO) )
207N/A {
207N/A logger.log(Level.INFO, "Progress: {0} ({1}%)",
207N/A new Object[] { lcur_count,
460N/A (lcur_count * 100.0f / est_total) });
460N/A }
460N/A
207N/A if (uidIter != null) {
207N/A String uid = Util.path2uid(path, DateTools
207N/A .timeToString(file.lastModified(),
312N/A DateTools.Resolution.MILLISECOND)); // construct uid for doc
207N/A while (uidIter.term() != null && uidIter.term().field().equals("u") &&
207N/A uidIter.term().text().compareTo(uid) < 0) {
207N/A removeFile();
207N/A uidIter.next();
207N/A }
207N/A
207N/A if (uidIter.term() != null && uidIter.term().field().equals("u") &&
207N/A uidIter.term().text().compareTo(uid) == 0) {
207N/A uidIter.next(); // keep matching docs
320N/A continue;
207N/A }
207N/A }
207N/A try {
207N/A addFile(file, path);
207N/A } catch (Exception e) {
207N/A logger.warning("Failed to add file '"
207N/A + file.getAbsolutePath() + "': " + e.getMessage());
207N/A }
207N/A }
207N/A }
459N/A }
580N/A
207N/A return lcur_count;
207N/A }
207N/A
207N/A /**
207N/A * Interrupt the index generation (and the index generation will stop as
207N/A * soon as possible)
459N/A */
508N/A public void interrupt() {
207N/A synchronized (lock) {
207N/A interrupted = true;
207N/A }
207N/A }
208N/A
208N/A private boolean isInterrupted() {
208N/A synchronized (lock) {
208N/A return interrupted;
208N/A }
208N/A }
208N/A
208N/A /**
208N/A * Register an object to receive events when modifications is done to the
208N/A * index database.
208N/A *
208N/A * @param listener the object to receive the events
208N/A */
208N/A public void addIndexChangedListener(IndexChangedListener listener) {
208N/A listeners.add(listener);
460N/A }
460N/A
460N/A /**
208N/A * Remove an object from the lists of objects to receive events when
208N/A * modifications is done to the index database
208N/A *
208N/A * @param listener the object to remove
208N/A */
208N/A public void removeIndexChangedListener(IndexChangedListener listener) {
208N/A listeners.remove(listener);
208N/A }
320N/A
320N/A /**
208N/A * List all files in all of the index databases
208N/A * @throws IOException if an error occurs
208N/A */
208N/A public static void listAllFiles() throws IOException {
208N/A listAllFiles(null);
274N/A }
274N/A
274N/A /**
274N/A * List all files in some of the index databases
274N/A * @param subFiles Subdirectories for the various projects to list the files
274N/A * for (or null or an empty list to dump all projects)
274N/A * @throws IOException if an error occurs
274N/A */
274N/A public static void listAllFiles(List<String> subFiles) throws IOException {
274N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
274N/A if (env.hasProjects()) {
274N/A if (subFiles == null || subFiles.isEmpty()) {
274N/A for (Project project : env.getProjects()) {
274N/A IndexDatabase db = new IndexDatabase(project);
274N/A db.listFiles();
274N/A }
274N/A } else {
274N/A for (String path : subFiles) {
274N/A Project project = Project.getProject(path);
460N/A if (project == null) {
274N/A logger.warning("Could not find a project for '"
274N/A + path + "'");
274N/A } else {
207N/A IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listFiles();
}
}
/**
* Print a listing of all of the files in this index database to stdout.
*
* @throws IOException If an IO error occurs while reading from the database
*/
@SuppressWarnings("resource")
public void listFiles() throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory); // open existing index
iter = ireader.terms(new Term("u", "")); // init uid iterator
while (iter.term() != null) {
System.out.println(Util.uid2url(iter.term().text()));
iter.next();
}
} finally {
IOUtils.close(iter);
IOUtils.close(ireader);
}
}
static void listFrequentTokens() throws IOException {
listFrequentTokens(null);
}
static void listFrequentTokens(List<String> subFiles) throws IOException {
final int limit = 4;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
logger.warning("Could not find a project for '" + path + "'");
} else {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listTokens(limit);
}
}
@SuppressWarnings("resource")
void listTokens(int freq) throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory);
iter = ireader.terms(new Term("defs", ""));
while (iter.term() != null) {
if (iter.term().field().startsWith("f")) {
if (iter.docFreq() > 16 && iter.term().text().length() > freq) {
System.out.println(iter.term().text());
}
iter.next();
} else {
break;
}
}
} finally {
IOUtils.close(iter);
IOUtils.close(ireader);
}
}
/**
* Get an indexReader for the Index database where a given file
* @param path the file to get the database for
* @return The index database where the file should be located or null if
* it cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
}
indexDir = new File(indexDir, p.getPath());
}
try {
@SuppressWarnings("resource")
FSDirectory fdir =
FSDirectory.open(indexDir,NoLockFactory.getNoLockFactory());
if (indexDir.exists() && IndexReader.indexExists(fdir)) {
ret = IndexReader.open(fdir);
}
} catch (Exception ex) {
logger.warning("Failed to open index '" + indexDir.getAbsolutePath() + "'");
logger.log(Level.FINE, "getIndexReader", ex);
}
return ret;
}
/**
* Get the latest definitions for a file from the index.
*
* @param file the file whose definitions to find
* @return definitions for the file, or {@code null} if they could not
* be found
* @throws IOException if an error happens when accessing the index
* @throws ParseException if an error happens when building the Lucene query
* @throws ClassNotFoundException if the class for the stored definitions
* instance cannot be found
*/
public static Definitions getDefinitions(File file)
throws IOException, ParseException, ClassNotFoundException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String path = env.getPathRelativeToSourceRoot(file, 0);
IndexReader ireader = getIndexReader(path);
if (ireader == null) {
// No index, no definitions...
return null;
}
try {
Query q = new QueryBuilder().setPath(path).build();
IndexSearcher searcher = new IndexSearcher(ireader);
try {
TopDocs top = searcher.search(q, 1);
if (top.totalHits == 0) {
// No hits, no definitions...
return null;
}
Document doc = searcher.doc(top.scoreDocs[0].doc);
String foundPath = doc.get("path");
// Only use the definitions if we found an exact match.
if (path.equals(foundPath)) {
Fieldable tags = doc.getFieldable("tags");
if (tags != null) {
return Definitions.deserialize(tags.getBinaryValue());
}
}
} finally {
searcher.close();
}
} finally {
ireader.close();
}
// Didn't find any definitions.
return null;
}
/**
* {@inheritDoc}
*/
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
IndexDatabase other = (IndexDatabase) obj;
return (this.project == other.project)
|| (this.project != null && this.project.equals(other.project));
}
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
int hash = 7;
hash = 41 * hash + (this.project == null ? 0 : this.project.hashCode());
return hash;
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return (project == null ? "" : project.getDescription()) + " Lucene IndexDB";
}
}