opengrok/index/IndexDatabase.java

	IndexDatabase.java revision 270
0N/A/*
0N/A * CDDL HEADER START
0N/A *
0N/A * The contents of this file are subject to the terms of the
0N/A * Common Development and Distribution License (the "License").
0N/A * You may not use this file except in compliance with the License.
0N/A *
0N/A * See LICENSE.txt included in this distribution for the specific
0N/A * language governing permissions and limitations under the License.
0N/A *
0N/A * When distributing Covered Code, include this CDDL HEADER in each
0N/A * file and include the License file at LICENSE.txt.
0N/A * If applicable, add the following below this CDDL HEADER, with the
0N/A * fields enclosed by brackets "[]" replaced with your own identifying
0N/A * information: Portions Copyright [yyyy] [name of copyright owner]
0N/A *
0N/A * CDDL HEADER END
0N/A */
0N/A
0N/A/*
0N/A * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
0N/A * Use is subject to license terms.
0N/A */
0N/Apackage org.opensolaris.opengrok.index;
0N/A
0N/Aimport java.io.BufferedInputStream;
0N/Aimport java.io.File;
0N/Aimport java.io.FileInputStream;
0N/Aimport java.io.FileNotFoundException;
0N/Aimport java.io.IOException;
0N/Aimport java.io.InputStream;
0N/Aimport java.util.ArrayList;
0N/Aimport java.util.Arrays;
0N/Aimport java.util.List;
0N/Aimport java.util.concurrent.ExecutorService;
0N/Aimport org.apache.lucene.document.DateTools;
0N/Aimport org.apache.lucene.document.Document;
0N/Aimport org.apache.lucene.index.IndexReader;
0N/Aimport org.apache.lucene.index.IndexWriter;
0N/Aimport org.apache.lucene.index.Term;
0N/Aimport org.apache.lucene.index.TermEnum;
0N/Aimport org.apache.lucene.search.spell.LuceneDictionary;
0N/Aimport org.apache.lucene.search.spell.SpellChecker;
0N/Aimport org.apache.lucene.store.FSDirectory;
0N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
0N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer;
0N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
0N/Aimport org.opensolaris.opengrok.configuration.Project;
0N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
0N/Aimport org.opensolaris.opengrok.web.Util;
0N/A
0N/A/**
0N/A * This class is used to create / update the index databases. Currently we use
0N/A * one index database per project.
0N/A *
0N/A * @author Trond Norbye
0N/A */
0N/Apublic class IndexDatabase {
0N/A
0N/A    private Project project;
0N/A    private FSDirectory indexDirectory;
0N/A    private FSDirectory spellDirectory;
0N/A    private IndexWriter writer;
0N/A    private IndexReader reader;
0N/A    private TermEnum uidIter;
0N/A    private IgnoredNames ignoredNames;
0N/A    private AnalyzerGuru analyzerGuru;
0N/A    private File xrefDir;
0N/A    private boolean interrupted;
0N/A    private List<IndexChangedListener> listeners;
0N/A    private File dirtyFile;
0N/A    private boolean dirty;
0N/A
0N/A    /**
0N/A     * Create a new instance of the Index Database. Use this constructor if
0N/A     * you don't use any projects
0N/A     *
0N/A     * @throws java.io.IOException if an error occurs while creating directories
0N/A     */
0N/A    public IndexDatabase() throws IOException {
0N/A        initialize();
0N/A    }
0N/A
0N/A    /**
0N/A     * Create a new instance of an Index Database for a given project
0N/A     * @param project the project to create the database for
0N/A     * @throws java.io.IOException if an errror occurs while creating directories
0N/A     */
0N/A    public IndexDatabase(Project project) throws IOException {
0N/A        this.project = project;
0N/A        initialize();
0N/A    }
0N/A
0N/A    /**
0N/A     * Update the index database for all of the projects. Print progress to
0N/A     * standard out.
0N/A     * @param executor An executor to run the job
0N/A     * @throws java.lang.Exception if an error occurs
0N/A     */
0N/A    public static void updateAll(ExecutorService executor) throws Exception {
0N/A        updateAll(executor, null);
0N/A    }
0N/A
0N/A    /**
0N/A     * Update the index database for all of the projects
0N/A     * @param executor An executor to run the job
0N/A     * @param listener where to signal the changes to the database
0N/A     * @throws java.lang.Exception if an error occurs
0N/A     */
0N/A    static void updateAll(ExecutorService executor, IndexChangedListener listener) throws Exception {
0N/A        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
0N/A        if (env.hasProjects()) {
0N/A            for (Project project : env.getProjects()) {
0N/A                final IndexDatabase db = new IndexDatabase(project);
0N/A                if (listener != null) {
0N/A                    db.addIndexChangedListener(listener);
0N/A                }
0N/A                executor.submit(new Runnable() {
0N/A
0N/A                    public void run() {
0N/A                        try {
0N/A                            db.update();
0N/A                        } catch (Exception e) {
0N/A                            e.printStackTrace();
0N/A                        }
0N/A                    }
0N/A                });
0N/A            }
0N/A        } else {
0N/A            final IndexDatabase db = new IndexDatabase();
0N/A            if (listener != null) {
0N/A                db.addIndexChangedListener(listener);
0N/A            }
0N/A
0N/A            executor.submit(new Runnable() {
0N/A
0N/A                public void run() {
0N/A                    try {
0N/A                        db.update();
0N/A                    } catch (Exception e) {
0N/A                        e.printStackTrace();
0N/A                    }
0N/A                }
0N/A            });
0N/A        }
0N/A
0N/A    }
0N/A
0N/A    private void initialize() throws IOException {
0N/A        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
0N/A        File indexDir = new File(env.getDataRootFile(), "index");
        File spellDir = new File(env.getDataRootFile(), "spellIndex");
        if (project != null) {
            indexDir = new File(indexDir, project.getPath());
            spellDir = new File(spellDir, project.getPath());
        }

        if (!indexDir.exists() || !spellDir.exists()) {
            indexDir.mkdirs();
            spellDir.mkdirs();
            // to avoid race conditions, just recheck..
            if (!indexDir.exists()) {
                throw new FileNotFoundException("Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
            }
            if (!spellDir.exists()) {
                throw new FileNotFoundException("Failed to create root directory [" + spellDir.getAbsolutePath() + "]");
            }
        }

        indexDirectory = FSDirectory.getDirectory(indexDir);
        spellDirectory = FSDirectory.getDirectory(spellDir);
        ignoredNames = env.getIgnoredNames();
        analyzerGuru = new AnalyzerGuru();
        if (RuntimeEnvironment.getInstance().isGenerateHtml()) {
            xrefDir = new File(env.getDataRootFile(), "xref");
        }
        listeners = new ArrayList<IndexChangedListener>();
        dirtyFile = new File(indexDir, "dirty");
        dirty = dirtyFile.exists();
    }

    /**
     * Update the content of this index database
     * @throws java.lang.Exception if an error occurs
     */
    public synchronized void update() throws Exception {
        interrupted = false;
        try {
            writer = new IndexWriter(indexDirectory, AnalyzerGuru.getAnalyzer());
            writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
            String root;
            File sourceRoot;

            if (project != null) {
                root = project.getPath();
                sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), project.getPath());
            } else {
                root = "";
                sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
            }

            String startuid = Util.uid(root, "");
            reader = IndexReader.open(indexDirectory);       // open existing index
            uidIter = reader.terms(new Term("u", startuid)); // init uid iterator

            indexDown(sourceRoot, root);

            while (uidIter.term() != null && uidIter.term().field().equals("u") && uidIter.term().text().startsWith(startuid)) {
                removeFile();
                uidIter.next();
            }
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e) {
                }
            }
            if (writer != null) {
                try {
                    writer.close();
                } catch (IOException e) {
                }
            }
        }

        if (!interrupted && dirty) {
            if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
                optimize();
            }
            createSpellingSuggestions();
        }
    }

    /**
     * Optimize all index databases
     * @param executor An executor to run the job
     * @throws java.lang.Exception if an error occurs
     */
    static void optimizeAll(ExecutorService executor) throws Exception {
        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
        if (env.hasProjects()) {
            for (Project project : env.getProjects()) {
                final IndexDatabase db = new IndexDatabase(project);
                if (db.dirty) {
                    executor.submit(new Runnable() {

                        public void run() {
                            try {
                                db.optimize();
                            } catch (Exception e) {
                                e.printStackTrace();
                            }
                        }
                    });
                }
            }
        } else {
            final IndexDatabase db = new IndexDatabase();
            if (db.dirty) {
                executor.submit(new Runnable() {

                    public void run() {
                        try {
                            db.update();
                        } catch (Exception e) {
                            e.printStackTrace();
                        }
                    }
                });
            }
        }
    }

    /**
     * Optimize the index database
     */
    public void optimize() {
        IndexWriter wrt = null;
        try {
            if (RuntimeEnvironment.getInstance().isVerbose()) {
                System.out.print("Optimizing the index ... ");
            }
            wrt = new IndexWriter(indexDirectory, null, false);
            wrt.optimize();
            if (RuntimeEnvironment.getInstance().isVerbose()) {
                System.out.println("done");
            }
            dirtyFile.delete();
            dirty = false;
        } catch (IOException e) {
            System.err.println("ERROR: optimizing index: " + e);
        } finally {
            if (wrt != null) {
                try {
                    wrt.close();
                } catch (IOException e) {
                }
            }

        }
    }

    /**
     * Generate a spelling suggestion for the definitions stored in defs
     */
    public void createSpellingSuggestions() {
        IndexReader indexReader = null;
        SpellChecker checker = null;

        try {
            if (RuntimeEnvironment.getInstance().isVerbose()) {
                System.out.print("Generating spelling suggestion index ... ");
            }
            indexReader = IndexReader.open(indexDirectory);
            checker = new SpellChecker(spellDirectory);
            checker.indexDictionary(new LuceneDictionary(indexReader, "defs"));
            if (RuntimeEnvironment.getInstance().isVerbose()) {
                System.out.println("done");
            }
        } catch (IOException e) {
            System.err.println("ERROR: Generating spelling: " + e);
        } finally {
            if (indexReader != null) {
                try {
                    indexReader.close();
                } catch (IOException e) {
                }
            }
            if (spellDirectory != null) {
                spellDirectory.close();
            }
        }
    }

    private void setDirty() {
        try {
            if (!dirty) {
                dirtyFile.createNewFile();
                dirty = true;
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    /**
     * Remove a stale file (uidIter.term().text()) from the index database
     * (and the xref file)
     * @throws java.io.IOException if an error occurs
     */
    private void removeFile() throws IOException {
        String path = Util.uid2url(uidIter.term().text());

        for (IndexChangedListener listener : listeners) {
            listener.fileRemoved(path);
        }
        writer.deleteDocuments(uidIter.term());

        File xrefFile = new File(xrefDir, path);
        xrefFile.delete();
        xrefFile.getParentFile().delete();
        setDirty();
    }

    /**
     * Add a file to the Lucene index (and generate a xref file)
     * @param file The file to add
     * @param path The path to the file (from source root)
     * @throws java.io.IOException if an error occurs
     */
    private void addFile(File file, String path) throws IOException {
        InputStream in;
        try {
            in = new BufferedInputStream(new FileInputStream(file));
        } catch (IOException ex) {
            System.err.println("Warning: " + ex.getMessage());
            return;
        }
        FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path);

        for (IndexChangedListener listener : listeners) {
            listener.fileAdded(path, fa.getClass().getSimpleName());
        }

        Document d = analyzerGuru.getDocument(file, in, path, fa);
        if (d != null) {
            writer.addDocument(d, fa);
            Genre g = fa.getFactory().getGenre();
            if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
                File xrefFile = new File(xrefDir, path);
                xrefFile.getParentFile().mkdirs();
                fa.writeXref(xrefDir, path);
            }
            setDirty();
        } else {
            System.err.println("Warning: did not add " + path);
        }

        try { in.close(); } catch (Exception e) {}
    }

    /**
     * Check if I should accept this file into the index database
     * @param file the file to check
     * @return true if the file should be included, false otherwise
     */
    private boolean accept(File file) {
        if (ignoredNames.ignore(file)) {
            return false;
        }

        if (!file.canRead()) {
            System.err.println("Warning: could not read " + file.getAbsolutePath());
            return false;
        }

        try {
            if (!file.getAbsolutePath().equals(file.getCanonicalPath())) {
                System.err.println("Warning: ignored link " + file.getAbsolutePath() +
                        " -> " + file.getCanonicalPath());
                return false;
            }
        } catch (IOException exp) {
            System.err.println("Warning: Failed to resolve name: " + file.getAbsolutePath());
            exp.printStackTrace();
        }

        return true;
    }

    /**
     * Generate indexes recursively
     * @param dir the root indexDirectory to generate indexes for
     * @param path the path
     */
    private void indexDown(File dir, String parent) throws IOException {
        if (interrupted) {
            return;
        }

        if (!accept(dir)) {
            return;
        }

        File[] files = dir.listFiles();
        if (files == null) {
            System.err.println("Failed to get file listing for: " + dir.getAbsolutePath());
            return;
        }
        Arrays.sort(files);

        for (File file : files) {
            if (accept(file)) {
                String path = parent + '/' + file.getName();
                if (file.isDirectory()) {
                    indexDown(file, path);
                } else {
                    if (uidIter != null) {
                        String uid = Util.uid(path, DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND));  // construct uid for doc
                        while (uidIter.term() != null && uidIter.term().field().equals("u") &&
                                uidIter.term().text().compareTo(uid) < 0) {
                            removeFile();
                            uidIter.next();
                        }

                        if (uidIter.term() != null && uidIter.term().field().equals("u") &&
                                uidIter.term().text().compareTo(uid) == 0) {
                            uidIter.next();        // keep matching docs
                        } else {
                            addFile(file, path);
                        }
                    } else {
                        addFile(file, path);
                    }
                }
            }
        }
    }

    /**
     * Interrupt the index generation (and the index generation will stop as
     * soon as possible)
     */
    public void interrupt() {
        interrupted = true;
    }

    /**
     * Register an object to receive events when modifications is done to the
     * index database.
     *
     * @param listener the object to receive the events
     */
    void addIndexChangedListener(IndexChangedListener listener) {
        listeners.add(listener);
    }

    /**
     * Remove an object from the lists of objects to receive events when
     * modifications is done to the index database
     *
     * @param listener the object to remove
     */
    void removeIndexChangedListener(IndexChangedListener listener) {
        listeners.remove(listener);
    }

    /**
     * List all files in all of the index databases
     * @throws java.lang.Exception if an error occurs
     */
    public static void listAllFiles() throws Exception {
        listAllFiles(null);
    }

    /**
     * List all files in some of the index databases
     * @param subFiles Subdirectories for the various projects to list the files
     *                 for (or null or an empty list to dump all projects)
     * @throws java.lang.Exception if an error occurs
     */
    public static void listAllFiles(List<String> subFiles) throws Exception {
        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
        if (!env.hasProjects()) {
            IndexDatabase db = new IndexDatabase();
            db.listFiles();
        } else {
            if (subFiles == null || subFiles.isEmpty()) {
                for (Project project : env.getProjects()) {
                    IndexDatabase db = new IndexDatabase(project);
                    db.listFiles();
                }
            } else {
                for (String path : subFiles) {
                    Project project = Project.getProject(path);
                    if (project == null) {
                        System.err.println("Warning: Could not find a project for \"" + path + "\"");
                    } else {
                        IndexDatabase db = new IndexDatabase(project);
                        db.listFiles();
                    }
                }
            }
        }
    }

    /**
     * List all of the files in this index database
     *
     * @throws java.lang.Exception if an error occurs
     */
    public void listFiles() throws Exception {
        IndexReader ireader = null;
        TermEnum iter = null;

        try {
            ireader = IndexReader.open(indexDirectory);       // open existing index
            iter = ireader.terms(new Term("u", "")); // init uid iterator
            while (iter.term() != null) {
                System.out.println(Util.uid2url(iter.term().text()));
                iter.next();
            }
        } finally {
            if (iter != null) {
                try {
                    iter.close();
                } catch (Exception e) {
                }
            }

            if (ireader != null) {
                try {
                    ireader.close();
                } catch (Exception e) {
                }
            }
        }
    }

    static void listFrequentTokens() throws Exception {
        listFrequentTokens(null);
    }

    static void listFrequentTokens(ArrayList<String> subFiles) throws Exception {
        final int limit = 4;

        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
        if (!env.hasProjects()) {
            IndexDatabase db = new IndexDatabase();
            db.listTokens(limit);
        } else {
            if (subFiles == null || subFiles.isEmpty()) {
                for (Project project : env.getProjects()) {
                    IndexDatabase db = new IndexDatabase(project);
                    db.listTokens(4);
                }
            } else {
                for (String path : subFiles) {
                    Project project = Project.getProject(path);
                    if (project == null) {
                        System.err.println("Warning: Could not find a project for \"" + path + "\"");
                    } else {
                        IndexDatabase db = new IndexDatabase(project);
                        db.listTokens(4);
                    }
                }
            }
        }
    }

    public void listTokens(int freq) throws Exception {
        IndexReader ireader = null;
        TermEnum iter = null;

        try {
            ireader = IndexReader.open(indexDirectory);
            iter = ireader.terms(new Term("defs", ""));
            while (iter.term() != null) {
                if (iter.term().field().startsWith("f")) {
                    if (iter.docFreq() > 16 && iter.term().text().length() > freq) {
                        System.out.println(iter.term().text());
                    }
                    iter.next();
                } else {
                    break;
                }
            }
        } finally {
            if (iter != null) {
                try {
                    iter.close();
                } catch (Exception e) {
                }
            }

            if (ireader != null) {
                try {
                    ireader.close();
                } catch (Exception e) {
                }
            }
        }
    }

    /**
     * Get an indexReader for the Index database where a given file
     * @param path the file to get the database for
     * @return The index database where the file should be located or null if
     *         it cannot be located.
     */
    public static IndexReader getIndexReader(String path) {
        IndexReader ret = null;

        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
        File indexDir = new File(env.getDataRootFile(), "index");

        if (env.hasProjects()) {
            Project p = Project.getProject(path);
            if (p != null) {
                indexDir = new File(indexDir, p.getPath());
            } else {
                return null;
            }
        }

        if (indexDir.exists() && IndexReader.indexExists(indexDir)) {
            try {
                ret = IndexReader.open(indexDir);
            } catch (Exception ex) {
                System.err.println("Failed to open index: " + indexDir.getAbsolutePath());
                ex.printStackTrace();
            }
        }

        return ret;
    }
}