IndexDatabase.java revision 1016
58N/A/*
58N/A * CDDL HEADER START
58N/A *
58N/A * The contents of this file are subject to the terms of the
58N/A * Common Development and Distribution License (the "License").
58N/A * You may not use this file except in compliance with the License.
58N/A *
58N/A * See LICENSE.txt included in this distribution for the specific
58N/A * language governing permissions and limitations under the License.
58N/A *
58N/A * When distributing Covered Code, include this CDDL HEADER in each
58N/A * file and include the License file at LICENSE.txt.
58N/A * If applicable, add the following below this CDDL HEADER, with the
58N/A * fields enclosed by brackets "[]" replaced with your own identifying
58N/A * information: Portions Copyright [yyyy] [name of copyright owner]
58N/A *
58N/A * CDDL HEADER END
58N/A */
58N/A
58N/A/*
1291N/A * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
1356N/A * Use is subject to license terms.
58N/A */
58N/Apackage org.opensolaris.opengrok.index;
58N/A
234N/Aimport java.io.BufferedInputStream;
234N/Aimport java.io.File;
234N/Aimport java.io.FileInputStream;
234N/Aimport java.io.FileNotFoundException;
1287N/Aimport java.io.IOException;
639N/Aimport java.io.InputStream;
639N/Aimport java.util.ArrayList;
234N/Aimport java.util.Arrays;
234N/Aimport java.util.Comparator;
234N/Aimport java.util.List;
1289N/Aimport java.util.concurrent.ExecutorService;
234N/Aimport java.util.logging.Level;
639N/Aimport java.util.logging.Logger;
639N/Aimport org.apache.lucene.document.DateTools;
58N/Aimport org.apache.lucene.document.Document;
1185N/Aimport org.apache.lucene.index.IndexReader;
667N/Aimport org.apache.lucene.index.IndexWriter;
1185N/Aimport org.apache.lucene.index.Term;
1016N/Aimport org.apache.lucene.index.TermEnum;
1291N/Aimport org.apache.lucene.search.spell.LuceneDictionary;
58N/Aimport org.apache.lucene.search.spell.SpellChecker;
1185N/Aimport org.apache.lucene.store.FSDirectory;
1016N/Aimport org.apache.lucene.store.LockFactory;
1185N/Aimport org.apache.lucene.store.NoLockFactory;
664N/Aimport org.apache.lucene.store.SimpleFSLockFactory;
1026N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
112N/Aimport org.opensolaris.opengrok.analysis.Ctags;
1195N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer;
58N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
58N/Aimport org.opensolaris.opengrok.configuration.Project;
77N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
77N/Aimport org.opensolaris.opengrok.history.HistoryException;
77N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
77N/Aimport org.opensolaris.opengrok.web.Util;
58N/A
418N/A/**
1327N/A * This class is used to create / update the index databases. Currently we use
1327N/A * one index database per project.
1327N/A *
1327N/A * @author Trond Norbye
1327N/A * @author Lubos Kosco , update for lucene 3.0.0
1327N/A */
1356N/Apublic class IndexDatabase {
1356N/A
1356N/A private Project project;
58N/A private FSDirectory indexDirectory;
773N/A private FSDirectory spellDirectory;
773N/A private IndexWriter writer;
58N/A private TermEnum uidIter;
773N/A private IgnoredNames ignoredNames;
773N/A private AnalyzerGuru analyzerGuru;
773N/A private File xrefDir;
773N/A private boolean interrupted;
58N/A private List<IndexChangedListener> listeners;
773N/A private File dirtyFile;
773N/A private final Object lock = new Object();
773N/A private boolean dirty;
773N/A private boolean running;
58N/A private List<String> directories;
58N/A private static final Logger log = Logger.getLogger(IndexDatabase.class.getName());
58N/A private Ctags ctags;
664N/A private LockFactory lockfact;
58N/A
65N/A /**
894N/A * Create a new instance of the Index Database. Use this constructor if
77N/A * you don't use any projects
99N/A *
99N/A * @throws java.io.IOException if an error occurs while creating directories
1115N/A */
1115N/A public IndexDatabase() throws IOException {
125N/A this(null);
112N/A }
1026N/A
129N/A /**
1100N/A * Create a new instance of an Index Database for a given project
129N/A * @param project the project to create the database for
129N/A * @throws java.io.IOException if an errror occurs while creating directories
318N/A */
318N/A public IndexDatabase(Project project) throws IOException {
144N/A this.project = project;
173N/A lockfact = new SimpleFSLockFactory();
253N/A initialize();
296N/A }
335N/A
480N/A /**
816N/A * Update the index database for all of the projects. Print progress to
816N/A * standard out.
833N/A * @param executor An executor to run the job
833N/A * @throws IOException if an error occurs
1416N/A */
1185N/A public static void updateAll(ExecutorService executor) throws IOException {
1016N/A updateAll(executor, null);
1123N/A }
1125N/A
1218N/A /**
1185N/A * Update the index database for all of the projects
1326N/A * @param executor An executor to run the job
993N/A * @param listener where to signal the changes to the database
1185N/A * @throws IOException if an error occurs
1185N/A */
1190N/A static void updateAll(ExecutorService executor, IndexChangedListener listener) throws IOException {
1185N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
1185N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
1252N/A
1185N/A if (env.hasProjects()) {
1185N/A for (Project project : env.getProjects()) {
1185N/A dbs.add(new IndexDatabase(project));
1185N/A }
1185N/A } else {
1185N/A dbs.add(new IndexDatabase());
1185N/A }
1185N/A
1185N/A for (IndexDatabase d : dbs) {
1185N/A final IndexDatabase db = d;
1252N/A if (listener != null) {
1185N/A db.addIndexChangedListener(listener);
1185N/A }
1185N/A
1185N/A executor.submit(new Runnable() {
1185N/A
1185N/A public void run() {
993N/A try {
993N/A db.update();
993N/A } catch (Exception e) {
1185N/A log.log(Level.FINE,"Problem updating lucene index database: ",e);
993N/A }
993N/A }
937N/A });
58N/A }
58N/A }
816N/A
58N/A /**
58N/A * Update the index database for a number of sub-directories
773N/A * @param executor An executor to run the job
58N/A * @param listener where to signal the changes to the database
664N/A * @param paths
58N/A * @throws IOException if an error occurs
850N/A */
1327N/A public static void update(ExecutorService executor, IndexChangedListener listener, List<String> paths) throws IOException {
870N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
870N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
99N/A
1115N/A for (String path : paths) {
101N/A Project project = Project.getProject(path);
106N/A if (project == null && env.hasProjects()) {
112N/A log.warning("Could not find a project for \"" + path + "\"");
1026N/A } else {
129N/A IndexDatabase db;
129N/A
129N/A try {
875N/A if (project == null) {
318N/A db = new IndexDatabase();
1356N/A } else {
173N/A db = new IndexDatabase(project);
253N/A }
296N/A
335N/A int idx = dbs.indexOf(db);
480N/A if (idx != -1) {
816N/A db = dbs.get(idx);
816N/A }
993N/A
1016N/A if (db.addDirectory(path)) {
1315N/A if (idx == -1) {
1185N/A dbs.add(db);
58N/A }
937N/A } else {
1185N/A log.warning("Directory does not exist \"" + path + "\"");
1185N/A }
1185N/A } catch (IOException e) {
1190N/A log.log(Level.WARNING, "An error occured while updating index", e);
1185N/A
1185N/A }
1185N/A }
1185N/A
1185N/A for (final IndexDatabase db : dbs) {
1185N/A db.addIndexChangedListener(listener);
1185N/A executor.submit(new Runnable() {
1185N/A
1185N/A public void run() {
1185N/A try {
1185N/A db.update();
1185N/A } catch (Exception e) {
1185N/A log.log(Level.WARNING, "An error occured while updating index", e);
1185N/A }
1190N/A }
1185N/A });
1185N/A }
1185N/A }
1185N/A }
1190N/A
58N/A @SuppressWarnings("PMD.CollapsibleIfStatements")
58N/A private void initialize() throws IOException {
58N/A synchronized (this) {
937N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
58N/A File indexDir = new File(env.getDataRootFile(), "index");
58N/A File spellDir = new File(env.getDataRootFile(), "spellIndex");
58N/A if (project != null) {
937N/A indexDir = new File(indexDir, project.getPath());
816N/A spellDir = new File(spellDir, project.getPath());
816N/A }
816N/A
816N/A if (!indexDir.exists() && !indexDir.mkdirs()) {
816N/A // to avoid race conditions, just recheck..
816N/A if (!indexDir.exists()) {
816N/A throw new FileNotFoundException("Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
816N/A }
816N/A }
816N/A
816N/A if (!spellDir.exists() && !spellDir.mkdirs()) {
816N/A if (!spellDir.exists()) {
816N/A throw new FileNotFoundException("Failed to create root directory [" + spellDir.getAbsolutePath() + "]");
816N/A }
816N/A }
773N/A
773N/A if (!env.isUsingLuceneLocking()) {
773N/A lockfact = NoLockFactory.getNoLockFactory();
773N/A }
773N/A indexDirectory = FSDirectory.open(indexDir,lockfact);
773N/A spellDirectory = FSDirectory.open(spellDir,lockfact);
58N/A ignoredNames = env.getIgnoredNames();
58N/A analyzerGuru = new AnalyzerGuru();
58N/A if (env.isGenerateHtml()) {
773N/A xrefDir = new File(env.getDataRootFile(), "xref");
773N/A }
773N/A listeners = new ArrayList<IndexChangedListener>();
773N/A dirtyFile = new File(indexDir, "dirty");
773N/A dirty = dirtyFile.exists();
58N/A directories = new ArrayList<String>();
58N/A }
58N/A }
773N/A
773N/A /**
773N/A * By default the indexer will traverse all directories in the project.
773N/A * If you add directories with this function update will just process
773N/A * the specified directories.
773N/A *
773N/A * @param dir The directory to scan
773N/A * @return <code>true</code> if the file is added, false oth
773N/A */
58N/A @SuppressWarnings("PMD.UseStringBufferForStringAppends")
58N/A public boolean addDirectory(String dir) {
58N/A String directory = dir;
773N/A if (directory.startsWith("\\")) {
773N/A directory = directory.replace('\\', '/');
773N/A } else if (directory.charAt(0) != '/') {
773N/A directory = "/" + directory;
773N/A }
773N/A File file = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), directory);
773N/A if (file.exists()) {
58N/A directories.add(directory);
58N/A return true;
58N/A } else {
773N/A return false;
773N/A }
773N/A }
773N/A
773N/A /**
773N/A * Update the content of this index database
773N/A * @throws IOException if an error occurs
773N/A * @throws HistoryException if an error occurs when accessing the history
773N/A */
773N/A public void update() throws IOException, HistoryException {
773N/A synchronized (lock) {
773N/A if (running) {
773N/A throw new IOException("Indexer already running!");
773N/A }
773N/A running = true;
773N/A interrupted = false;
773N/A }
773N/A
773N/A String ctgs = RuntimeEnvironment.getInstance().getCtags();
773N/A if (ctgs != null) {
773N/A ctags = new Ctags();
773N/A ctags.setBinary(ctgs);
773N/A }
937N/A if (ctags == null) {
58N/A log.severe("Unable to run ctags! searching definitions will not work!");
58N/A }
58N/A
937N/A try {
58N/A //TODO we might need to add writer.commit after certain phases of index generation, right now it will only happen in the end
58N/A writer = new IndexWriter(indexDirectory, AnalyzerGuru.getAnalyzer(),IndexWriter.MaxFieldLength.UNLIMITED);
58N/A writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
937N/A
58N/A if (directories.isEmpty()) {
58N/A if (project == null) {
58N/A directories.add("");
937N/A } else {
58N/A directories.add(project.getPath());
58N/A }
58N/A }
937N/A
58N/A for (String dir : directories) {
58N/A File sourceRoot;
58N/A if ("".equals(dir)) {
937N/A sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
58N/A } else {
58N/A sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
58N/A }
937N/A
664N/A HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
58N/A
58N/A String startuid = Util.uid(dir, "");
937N/A IndexReader reader = IndexReader.open(indexDirectory,false); // open existing index
664N/A try {
58N/A uidIter = reader.terms(new Term("u", startuid)); // init uid iterator
58N/A
937N/A indexDown(sourceRoot, dir);
58N/A
58N/A while (uidIter.term() != null && uidIter.term().field().equals("u") && uidIter.term().text().startsWith(startuid)) {
58N/A removeFile();
937N/A uidIter.next();
1185N/A }
1252N/A } finally {
1252N/A reader.close();
1252N/A }
1252N/A }
1185N/A } finally {
1185N/A if (writer != null) {
58N/A try {
58N/A writer.close();
58N/A } catch (IOException e) {
937N/A log.log(Level.WARNING, "An error occured while closing writer", e);
65N/A }
65N/A }
65N/A
937N/A if (ctags != null) {
65N/A try {
65N/A ctags.close();
65N/A } catch (IOException e) {
937N/A log.log(Level.WARNING, "An error occured while closing ctags process", e);
77N/A }
77N/A }
77N/A
937N/A synchronized (lock) {
77N/A running = false;
77N/A }
77N/A }
99N/A
99N/A if (!isInterrupted() && isDirty()) {
99N/A if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
99N/A optimize();
99N/A }
99N/A createSpellingSuggestions();
99N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
99N/A File timestamp = new File(env.getDataRootFile(), "timestamp");
99N/A if (timestamp.exists()) {
99N/A if (!timestamp.setLastModified(System.currentTimeMillis())) {
99N/A log.warning("Failed to set last modified time on '" + timestamp.getAbsolutePath() + "', used for timestamping the index database.");
99N/A }
99N/A } else {
99N/A if (!timestamp.createNewFile()) {
99N/A log.warning("Failed to create file '" + timestamp.getAbsolutePath() + "', used for timestamping the index database.");
99N/A }
937N/A }
1115N/A }
1115N/A }
1115N/A
1115N/A /**
1115N/A * Optimize all index databases
1115N/A * @param executor An executor to run the job
1115N/A * @throws IOException if an error occurs
1115N/A */
125N/A static void optimizeAll(ExecutorService executor) throws IOException {
125N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
125N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
937N/A if (env.hasProjects()) {
125N/A for (Project project : env.getProjects()) {
125N/A dbs.add(new IndexDatabase(project));
125N/A }
106N/A } else {
106N/A dbs.add(new IndexDatabase());
937N/A }
106N/A
106N/A for (IndexDatabase d : dbs) {
106N/A final IndexDatabase db = d;
937N/A if (db.isDirty()) {
106N/A executor.submit(new Runnable() {
106N/A
106N/A public void run() {
112N/A try {
112N/A db.update();
112N/A } catch (Exception e) {
112N/A log.log(Level.FINE,"Problem updating lucene index database: ",e);
112N/A }
112N/A }
112N/A });
112N/A }
129N/A }
1026N/A }
1026N/A
1026N/A /**
1026N/A * Optimize the index database
1026N/A */
1026N/A public void optimize() {
1026N/A synchronized (lock) {
1026N/A if (running) {
129N/A log.warning("Optimize terminated... Someone else is updating / optimizing it!");
129N/A return ;
129N/A }
129N/A running = true;
129N/A }
129N/A IndexWriter wrt = null;
129N/A try {
129N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
1100N/A log.info("Optimizing the index ... ");
1100N/A }
1100N/A wrt = new IndexWriter(indexDirectory, null, false,IndexWriter.MaxFieldLength.UNLIMITED);
1100N/A wrt.optimize();
1100N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
1100N/A log.info("done");
1100N/A }
1100N/A synchronized (lock) {
129N/A if (dirtyFile.exists() && !dirtyFile.delete()) {
129N/A log.fine("Failed to remove \"dirty-file\": " +
129N/A dirtyFile.getAbsolutePath());
129N/A }
129N/A dirty = false;
129N/A }
129N/A } catch (IOException e) {
129N/A log.severe("ERROR: optimizing index: " + e);
129N/A } finally {
129N/A if (wrt != null) {
129N/A try {
129N/A wrt.close();
129N/A } catch (IOException e) {
129N/A log.log(Level.WARNING, "An error occured while closing writer", e);
129N/A }
937N/A }
318N/A synchronized (lock) {
318N/A running = false;
318N/A }
318N/A }
318N/A }
318N/A
318N/A /**
318N/A * Generate a spelling suggestion for the definitions stored in defs
318N/A */
318N/A public void createSpellingSuggestions() {
318N/A IndexReader indexReader = null;
318N/A SpellChecker checker = null;
318N/A
318N/A try {
318N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
144N/A log.info("Generating spelling suggestion index ... ");
144N/A }
144N/A indexReader = IndexReader.open(indexDirectory,false);
144N/A checker = new SpellChecker(spellDirectory);
144N/A //TODO below seems only to index "defs" , possible bug ?
144N/A checker.indexDictionary(new LuceneDictionary(indexReader, "defs"));
144N/A if (RuntimeEnvironment.getInstance().isVerbose()) {
144N/A log.info("done");
173N/A }
173N/A } catch (IOException e) {
173N/A log.severe("ERROR: Generating spelling: " + e);
173N/A } finally {
173N/A if (indexReader != null) {
173N/A try {
173N/A indexReader.close();
173N/A } catch (IOException e) {
234N/A log.log(Level.WARNING, "An error occured while closing reader", e);
253N/A }
253N/A }
253N/A if (spellDirectory != null) {
253N/A spellDirectory.close();
253N/A }
253N/A }
253N/A }
253N/A
296N/A private boolean isDirty() {
296N/A synchronized (lock) {
296N/A return dirty;
296N/A }
296N/A }
296N/A
296N/A private void setDirty() {
335N/A synchronized (lock) {
335N/A try {
335N/A if (!dirty && !dirtyFile.createNewFile()) {
335N/A if (!dirtyFile.exists()) {
335N/A log.log(Level.FINE,
335N/A "Failed to create \"dirty-file\": {0}",
335N/A dirtyFile.getAbsolutePath());
335N/A }
335N/A dirty = true;
480N/A }
480N/A } catch (IOException e) {
480N/A log.log(Level.FINE,"When creating dirty file: ",e);
480N/A }
480N/A }
480N/A }
480N/A /**
937N/A * Remove a stale file (uidIter.term().text()) from the index database
1330N/A * (and the xref file)
1330N/A * @throws java.io.IOException if an error occurs
1330N/A */
1330N/A private void removeFile() throws IOException {
1330N/A String path = Util.uid2url(uidIter.term().text());
1330N/A
1330N/A for (IndexChangedListener listener : listeners) {
1330N/A listener.fileRemoved(path);
1330N/A }
1330N/A writer.deleteDocuments(uidIter.term());
1330N/A
1330N/A File xrefFile;
1330N/A if (RuntimeEnvironment.getInstance().isCompressXref()) {
1330N/A xrefFile = new File(xrefDir, path + ".gz");
1330N/A } else {
1330N/A xrefFile = new File(xrefDir, path);
1330N/A }
1330N/A File parent = xrefFile.getParentFile();
1330N/A
1330N/A if (!xrefFile.delete() && xrefFile.exists()) {
1330N/A log.info("Failed to remove obsolete xref-file: " +
1330N/A xrefFile.getAbsolutePath());
1330N/A }
1330N/A
1330N/A // Remove the parent directory if it's empty
1324N/A if (parent.delete()) {
1324N/A log.fine("Removed empty xref dir:" + parent.getAbsolutePath());
1324N/A }
1324N/A
937N/A setDirty();
1330N/A }
1324N/A
1330N/A /**
1324N/A * Add a file to the Lucene index (and generate a xref file)
1330N/A * @param file The file to add
667N/A * @param path The path to the file (from source root)
833N/A * @throws java.io.IOException if an error occurs
1286N/A */
1324N/A private void addFile(File file, String path) throws IOException {
1286N/A final InputStream in =
1324N/A new BufferedInputStream(new FileInputStream(file));
1324N/A try {
1324N/A FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path);
1324N/A if (log.isLoggable(Level.FINER)) {
1324N/A log.finer("Adding file:"+path);}
1324N/A fa.setCtags(ctags);
1289N/A fa.setProject(Project.getProject(path));
1324N/A
1324N/A Document d;
1324N/A try {
1324N/A d = analyzerGuru.getDocument(file, in, path, fa);
1324N/A } catch (Exception e) {
1324N/A log.log(Level.INFO,
1324N/A "Skipped file ''{0}'' because the analyzer didn''t " +
1286N/A "understand it.",
1324N/A path);
1324N/A log.log(Level.FINE, "Exception from analyzer:", e);
1324N/A return;
1324N/A }
1327N/A
1327N/A writer.addDocument(d, fa);
1324N/A Genre g = fa.getFactory().getGenre();
1324N/A if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
1324N/A File xrefFile = new File(xrefDir, path);
1324N/A // If mkdirs() returns false, the failure is most likely
1324N/A // because the file already exists. But to check for the
1324N/A // file first and only add it if it doesn't exists would
1324N/A // only increase the file IO...
1324N/A if (!xrefFile.getParentFile().mkdirs()) {
1286N/A assert xrefFile.getParentFile().exists();
1324N/A }
1286N/A fa.writeXref(xrefDir, path);
1286N/A }
1299N/A setDirty();
1324N/A for (IndexChangedListener listener : listeners) {
1324N/A listener.fileAdded(path, fa.getClass().getSimpleName());
1324N/A }
1324N/A } finally {
1324N/A in.close();
1324N/A }
1324N/A }
1324N/A
1324N/A /**
1324N/A * Check if I should accept this file into the index database
1331N/A * @param file the file to check
1299N/A * @return true if the file should be included, false otherwise
1299N/A */
1324N/A private boolean accept(File file) {
1324N/A if (ignoredNames.ignore(file)) {
1324N/A return false;
1324N/A }
1299N/A
1299N/A String absolutePath = file.getAbsolutePath();
1299N/A
1324N/A if (!file.canRead()) {
1324N/A log.warning("Warning: could not read " + absolutePath);
1324N/A return false;
1324N/A }
1324N/A
1324N/A try {
1324N/A String canonicalPath = file.getCanonicalPath();
1324N/A if (!absolutePath.equals(canonicalPath) && !acceptSymlink(absolutePath, canonicalPath)) {
1324N/A log.fine("Skipped symlink '" + absolutePath +
1324N/A "' -> '" + canonicalPath + "'");
1331N/A return false;
1299N/A }
1299N/A //below will only let go files and directories, anything else is considered special and is not added
1324N/A if (!file.isFile() && !file.isDirectory()) {
1324N/A log.warning("Warning: ignored special file " + absolutePath);
1324N/A return false;
1324N/A }
1299N/A } catch (IOException exp) {
1299N/A log.warning("Warning: Failed to resolve name: " + absolutePath);
1326N/A log.log(Level.FINE,"Stack Trace: ",exp);
1331N/A }
1331N/A
1331N/A if (file.isDirectory()) {
1331N/A // always accept directories so that their files can be examined
1331N/A return true;
1331N/A }
1331N/A
1331N/A if (HistoryGuru.getInstance().hasHistory(file)) {
1331N/A // versioned files should always be accepted
1331N/A return true;
1331N/A }
1331N/A
1331N/A // this is an unversioned file, check if it should be indexed
1331N/A return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
1331N/A }
1331N/A
1331N/A /**
1331N/A * Check if I should accept the path containing a symlink
1331N/A * @param absolutePath the path with a symlink to check
1331N/A * @param canonicalPath the canonical path to the file
1326N/A * @return true if the file should be accepted, false otherwise
1326N/A */
1326N/A private boolean acceptSymlink(String absolutePath, String canonicalPath) throws IOException {
1326N/A for (String allowedSymlink : RuntimeEnvironment.getInstance().getAllowedSymlinks()) {
1326N/A if (absolutePath.startsWith(allowedSymlink)) {
1326N/A String allowedTarget = new File(allowedSymlink).getCanonicalPath();
1326N/A if (canonicalPath.startsWith(allowedTarget) &&
1326N/A absolutePath.substring(allowedSymlink.length()).equals(canonicalPath.substring(allowedTarget.length()))) {
1326N/A return true;
1326N/A }
1326N/A }
1326N/A }
1326N/A return false;
1326N/A }
1326N/A
1326N/A /**
1326N/A * Generate indexes recursively
1326N/A * @param dir the root indexDirectory to generate indexes for
1326N/A * @param path the path
1326N/A */
1326N/A private void indexDown(File dir, String parent) throws IOException {
1326N/A if (isInterrupted()) {
833N/A return;
833N/A }
833N/A
833N/A if (!accept(dir)) {
833N/A return;
833N/A }
833N/A
833N/A File[] files = dir.listFiles();
833N/A if (files == null) {
833N/A log.severe("Failed to get file listing for: " + dir.getAbsolutePath());
833N/A return;
833N/A }
833N/A Arrays.sort(files, new Comparator<File>() {
833N/A
833N/A public int compare(File p1, File p2) {
833N/A return p1.getName().compareTo(p2.getName());
1416N/A }
1416N/A });
1393N/A
1393N/A for (File file : files) {
1416N/A if (accept(file)) {
1416N/A String path = parent + '/' + file.getName();
1393N/A if (file.isDirectory()) {
1393N/A indexDown(file, path);
1016N/A } else {
1016N/A if (uidIter != null) {
1016N/A String uid = Util.uid(path, DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND)); // construct uid for doc
1016N/A while (uidIter.term() != null && uidIter.term().field().equals("u") &&
1016N/A uidIter.term().text().compareTo(uid) < 0) {
1016N/A removeFile();
1016N/A uidIter.next();
1016N/A }
1123N/A
1123N/A if (uidIter.term() != null && uidIter.term().field().equals("u") &&
1123N/A uidIter.term().text().compareTo(uid) == 0) {
1123N/A uidIter.next(); // keep matching docs
1123N/A continue;
1123N/A }
1123N/A }
1123N/A try {
1125N/A addFile(file, path);
1125N/A } catch (Exception e) {
1125N/A log.log(Level.WARNING,
1125N/A "Failed to add file " + file.getAbsolutePath(),
1125N/A e);
1125N/A }
1125N/A }
1330N/A }
234N/A }
234N/A }
234N/A
234N/A /**
234N/A * Interrupt the index generation (and the index generation will stop as
234N/A * soon as possible)
509N/A */
640N/A public void interrupt() {
640N/A synchronized (lock) {
640N/A interrupted = true;
1195N/A }
640N/A }
639N/A
937N/A private boolean isInterrupted() {
639N/A synchronized (lock) {
639N/A return interrupted;
639N/A }
639N/A }
639N/A
937N/A /**
639N/A * Register an object to receive events when modifications is done to the
640N/A * index database.
640N/A *
640N/A * @param listener the object to receive the events
234N/A */
234N/A public void addIndexChangedListener(IndexChangedListener listener) {
234N/A listeners.add(listener);
639N/A }
640N/A
640N/A /**
640N/A * Remove an object from the lists of objects to receive events when
1195N/A * modifications is done to the index database
640N/A *
639N/A * @param listener the object to remove
937N/A */
937N/A public void removeIndexChangedListener(IndexChangedListener listener) {
937N/A listeners.remove(listener);
639N/A }
639N/A
639N/A /**
639N/A * List all files in all of the index databases
639N/A * @throws IOException if an error occurs
639N/A */
937N/A public static void listAllFiles() throws IOException {
639N/A listAllFiles(null);
640N/A }
640N/A
640N/A /**
509N/A * List all files in some of the index databases
234N/A * @param subFiles Subdirectories for the various projects to list the files
234N/A * for (or null or an empty list to dump all projects)
640N/A * @throws IOException if an error occurs
234N/A */
234N/A public static void listAllFiles(List<String> subFiles) throws IOException {
639N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
58N/A if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
log.warning("Warning: Could not find a project for \"" + path + "\"");
} else {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listFiles();
}
}
/**
* List all of the files in this index database
*
* @throws IOException If an IO error occurs while reading from the database
*/
public void listFiles() throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory,false); // open existing index
iter = ireader.terms(new Term("u", "")); // init uid iterator
while (iter.term() != null) {
log.info(Util.uid2url(iter.term().text()));
iter.next();
}
} finally {
if (iter != null) {
try {
iter.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index iterator", e);
}
}
if (ireader != null) {
try {
ireader.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
}
}
static void listFrequentTokens() throws IOException {
listFrequentTokens(null);
}
static void listFrequentTokens(List<String> subFiles) throws IOException {
final int limit = 4;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
log.warning("Warning: Could not find a project for \"" + path + "\"");
} else {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listTokens(limit);
}
}
public void listTokens(int freq) throws IOException {
IndexReader ireader = null;
TermEnum iter = null;
try {
ireader = IndexReader.open(indexDirectory,false);
iter = ireader.terms(new Term("defs", ""));
while (iter.term() != null) {
if (iter.term().field().startsWith("f")) {
if (iter.docFreq() > 16 && iter.term().text().length() > freq) {
log.warning(iter.term().text());
}
iter.next();
} else {
break;
}
}
} finally {
if (iter != null) {
try {
iter.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index iterator", e);
}
}
if (ireader != null) {
try {
ireader.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
}
}
/**
* Get an indexReader for the Index database where a given file
* @param path the file to get the database for
* @return The index database where the file should be located or null if
* it cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
} else {
indexDir = new File(indexDir, p.getPath());
}
}
try {
FSDirectory fdir=FSDirectory.open(indexDir,NoLockFactory.getNoLockFactory());
if (indexDir.exists() && IndexReader.indexExists(fdir)) {
ret = IndexReader.open(fdir,false);
}
} catch (Exception ex) {
log.severe("Failed to open index: " + indexDir.getAbsolutePath());
log.log(Level.FINE,"Stack Trace: ",ex);
}
return ret;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final IndexDatabase other = (IndexDatabase) obj;
if (this.project != other.project && (this.project == null || !this.project.equals(other.project))) {
return false;
}
return true;
}
@Override
public int hashCode() {
int hash = 7;
hash = 41 * hash + (this.project == null ? 0 : this.project.hashCode());
return hash;
}
}