opengrok/index/Indexer.java

	Indexer.java revision 467
0N/A/*
0N/A * CDDL HEADER START
0N/A *
0N/A * The contents of this file are subject to the terms of the
407N/A * Common Development and Distribution License (the "License").
0N/A * You may not use this file except in compliance with the License.
0N/A *
0N/A * See LICENSE.txt included in this distribution for the specific
0N/A * language governing permissions and limitations under the License.
0N/A *
0N/A * When distributing Covered Code, include this CDDL HEADER in each
0N/A * file and include the License file at LICENSE.txt.
0N/A * If applicable, add the following below this CDDL HEADER, with the
0N/A * fields enclosed by brackets "[]" replaced with your own identifying
0N/A * information: Portions Copyright [yyyy] [name of copyright owner]
0N/A *
0N/A * CDDL HEADER END
0N/A */
0N/A
0N/A/*
0N/A * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
0N/A * Use is subject to license terms.
0N/A */
0N/Apackage org.opensolaris.opengrok.index;
0N/A
0N/Aimport java.io.BufferedReader;
335N/Aimport java.io.File;
335N/Aimport java.io.FileReader;
335N/Aimport java.io.IOException;
335N/Aimport java.net.InetAddress;
335N/Aimport java.text.ParseException;
335N/Aimport java.util.ArrayList;
335N/Aimport java.util.Collections;
335N/Aimport java.util.Comparator;
335N/Aimport java.util.List;
335N/Aimport java.util.concurrent.ExecutorService;
335N/Aimport java.util.concurrent.Executors;
335N/Aimport java.util.logging.Level;
335N/Aimport java.util.logging.Logger;
271N/Aimport org.opensolaris.opengrok.Info;
99N/Aimport org.opensolaris.opengrok.OpenGrokLogger;
0N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
0N/Aimport org.opensolaris.opengrok.configuration.Project;
0N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
0N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
0N/Aimport org.opensolaris.opengrok.util.Getopt;
0N/A
0N/A/**
0N/A * Creates and updates an inverted source index
0N/A * as well as generates Xref, file stats etc., if specified
0N/A * in the options
0N/A */
0N/A@SuppressWarnings({"PMD.AvoidPrintStackTrace","PMD.SystemPrintln"})
0N/Apublic final class Indexer {
0N/A
0N/A    private final static String ON = "on";
0N/A    private final static String OFF = "off";
0N/A
0N/A   private static Indexer index = new Indexer();
0N/A   private static final Logger log = Logger.getLogger(Indexer.class.getName());
271N/A
271N/A   public static Indexer getInstance() {
202N/A      return index;
202N/A   }
0N/A    /**
0N/A     * Program entry point
0N/A     * @param argv argument vector
0N/A     */
0N/A    @SuppressWarnings("PMD.UseStringBufferForStringAppends")
0N/A    public static void main(String argv[]) {
0N/A        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
0N/A        boolean runIndex = true;
0N/A        boolean update = true;
0N/A        boolean optimizedChanged = false;
202N/A        CommandLineOptions cmdOptions = new CommandLineOptions();
202N/A
202N/A        if (argv.length == 0) {
202N/A            System.err.println(cmdOptions.getUsage());
202N/A            System.exit(1);
202N/A        } else {
202N/A            boolean searchRepositories = false;
202N/A            ArrayList<String> subFiles = new ArrayList<String>();
202N/A            ArrayList<String> repositories = new ArrayList<String>();
0N/A            String configFilename = null;
202N/A            String configHost = null;
0N/A            boolean addProjects = false;
202N/A            boolean refreshHistory = false;
0N/A            String defaultProject = null;
0N/A            boolean listFiles = false;
202N/A            boolean createDict = false;
202N/A            int noThreads = Runtime.getRuntime().availableProcessors();
0N/A
0N/A            // Parse command line options:
0N/A            Getopt getopt = new Getopt(argv, cmdOptions.getCommandString());
0N/A
0N/A            try {
0N/A                getopt.parse();
0N/A            } catch (ParseException ex) {
58N/A                System.err.println("OpenGrok: " + ex.getMessage());
0N/A                System.err.println(cmdOptions.getUsage());
0N/A                System.exit(1);
0N/A            }
58N/A
99N/A            try{
99N/A                int cmd;
99N/A
99N/A                // We need to read the configuration file first, since we
0N/A                // will try to overwrite options..
0N/A                while ((cmd = getopt.getOpt()) != -1) {
0N/A                    if (cmd == 'R') {
0N/A                        env.readConfiguration(new File(getopt.getOptarg()));
0N/A                        break;
0N/A                    }
271N/A                }
0N/A
0N/A                // Now we can handle all the other options..
0N/A                getopt.reset();
0N/A                while ((cmd = getopt.getOpt()) != -1) {
0N/A                    switch (cmd) {
0N/A                    case 't':
335N/A                        createDict = true;
335N/A                        runIndex = false;
335N/A                        break;
271N/A
271N/A                    case 'q': env.setVerbose(false); break;
271N/A                    case 'e': env.setGenerateHtml(false); break;
271N/A                    case 'P': addProjects = true; break;
335N/A                    case 'p': defaultProject = getopt.getOptarg(); break;
335N/A                    case 'c': env.setCtags(getopt.getOptarg()); break;
335N/A                    case 'w': {
335N/A                        String webapp = getopt.getOptarg();
335N/A                        if (webapp.charAt(0) != '/' && !webapp.startsWith("http")) {
335N/A                            webapp = "/" + webapp;
335N/A                        }
335N/A                        if (webapp.endsWith("/")) {
0N/A                            env.setUrlPrefix(webapp + "s?");
0N/A                        } else {
0N/A                            env.setUrlPrefix(webapp + "/s?");
0N/A                        }
                    }
                    break;
                    case 'W': configFilename = getopt.getOptarg(); break;
                    case 'U': configHost = getopt.getOptarg(); break;
                    case 'R':
                        // already handled
                        break;
                    case 'n': runIndex = false; break;
                    case 'H': refreshHistory = true; break;
                    case 'h' : repositories.add(getopt.getOptarg()); break;
                    case 'r': {
                        if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                            env.setRemoteScmSupported(true);
                        } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                            env.setRemoteScmSupported(false);
                        } else {
                            System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -r");
                            System.err.println("       Ex: \"-r on\" will allow retrival for remote SCM systems");
                            System.err.println("           \"-r off\" will ignore SCM for remote systems");
                        }
                    }
                    break;
                    case 'O': {
                       boolean oldval = env.isOptimizeDatabase();
                        if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                            env.setOptimizeDatabase(true);
                        } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                            env.setOptimizeDatabase(false);
                        } else {
                            System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -O");
                            System.err.println("       Ex: \"-O on\" will optimize the database as part of the index generation");
                            System.err.println("           \"-O off\" disable optimization of the index database");
                        }
                       if (oldval != env.isOptimizeDatabase()) {
                           optimizedChanged = true;
                        }
                    }
                    break;
                    case 'v': env.setVerbose(true); break;

                    case 's': {
                        File file = new File(getopt.getOptarg());
                        if (!file.isDirectory()) {
                            System.err.println("ERROR: No such directory: " + file.toString());
                            System.exit(1);
                        }

                        env.setSourceRootFile(file);
                        break;
                    }
                    case 'd':
                        env.setDataRoot(getopt.getOptarg());
                        break;
                    case 'i':
                        env.getIgnoredNames().add(getopt.getOptarg());
                        break;
                    case 'S' : searchRepositories = true; break;
                    case 'Q' :
                        if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                            env.setQuickContextScan(true);
                        } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                            env.setQuickContextScan(false);
                        } else {
                            System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -Q");
                            System.err.println("       Ex: \"-Q on\" will just scan a \"chunk\" of the file and insert \"[..all..]\"");
                            System.err.println("           \"-Q off\" will try to build a more accurate list by reading the complete file.");
                        }

                        break;
                    case 'm' : {
                        try {
                            env.setIndexWordLimit(Integer.parseInt(getopt.getOptarg()));
                        } catch (NumberFormatException exp) {
                            System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());
                            System.exit(1);
                        }
                        break;
                    }
                    case 'a' :
                        if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                            env.setAllowLeadingWildcard(true);
                        } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                            env.setAllowLeadingWildcard(false);
                        } else {
                            System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -a");
                            System.err.println("       Ex: \"-a on\" will allow a search to start with a wildcard");
                            System.err.println("           \"-a off\" will disallow a search to start with a wildcard");
                            System.exit(1);
                        }

                        break;

                    case 'A': {
                            String[] arg = getopt.getOptarg().split(":");
                            if (arg.length != 2) {
                                System.err.println("ERROR: You must specify: -A extension:class");
                                System.err.println("       Ex: -A foo:org.opensolaris.opengrok.analysis.c.CAnalyzer");
                                System.err.println("           will use the C analyzer for all files ending with .foo");
                                System.err.println("       Ex: -A c:-");
                                System.err.println("           will disable the c-analyzer for for all files ending with .c");
                                System.exit(1);
                            }

                            arg[0] = arg[0].substring(arg[0].lastIndexOf('.') + 1).toUpperCase();
                            if (arg[1].equals("-")) {
                                AnalyzerGuru.addExtension(arg[0], null);
                                break;
                            }

                            try {
                                AnalyzerGuru.addExtension(
                                        arg[0],
                                        AnalyzerGuru.findFactory(arg[1]));
                            } catch (Exception e) {
                                System.err.println("Unable to use " + arg[1] +
                                                   " as a FileAnalyzerFactory");
                                e.printStackTrace();
                                System.exit(1);
                            }
                        }
                        break;
                    case 'L' :
                        env.setWebappLAF(getopt.getOptarg());
                        break;
                    case 'T' :
                        try {
                            noThreads = Integer.parseInt(getopt.getOptarg());
                        } catch (NumberFormatException exp) {
                            System.err.println("ERROR: Failed to parse argument to \"-T\": " + exp.getMessage());
                            System.exit(1);
                        }
                        break;
                    case 'l' :
                        if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                            env.setUsingLuceneLocking(true);
                        } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                            env.setUsingLuceneLocking(false);
                        } else {
                            System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -l");
                            System.err.println("       Ex: \"-l on\" will enable locks in Lucene");
                            System.err.println("           \"-l off\" will disable locks in Lucene");
                        }
                        break;
                    case 'V' :
                        System.out.println(Info.getFullVersion());
                        System.exit(0);
                        break;

                    case '?':
                        System.err.println(cmdOptions.getUsage());
                        System.exit(0);
                        break;

                    default:
                        System.err.println("Internal Error - Unimplemented cmdline option: " + (char)cmd);
                        System.exit(1);
                    }
                }

                int optind = getopt.getOptind();
                if (optind != -1) {
                    while (optind < argv.length) {
                        subFiles.add(argv[optind]);
                        ++optind;
                    }
                }

                getInstance().prepareIndexer(env, searchRepositories, addProjects,
                    defaultProject,configFilename,refreshHistory,
                    listFiles,createDict,subFiles,repositories);
            if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
                IndexChangedListener progress = new DefaultIndexChangedListener();
                getInstance().doIndexerExecution(update, noThreads, subFiles,
                        progress);
            }
            getInstance().sendToConfigHost(env, configHost);
         } catch (IndexerException ex) {
            OpenGrokLogger.getLogger().log(Level.SEVERE, "Exception running indexer", ex);
            System.err.println(cmdOptions.getUsage());
            System.exit(1);
         } catch (IOException ioe) {
            System.err.println("Got IOException " + ioe);
            OpenGrokLogger.getLogger().log(Level.SEVERE, "Exception running indexer", ioe);
            System.exit(1);
         }
      }

   }


   public void prepareIndexer(RuntimeEnvironment env,
           boolean searchRepositories,
           boolean addProjects,
           String defaultProject,
           String configFilename,
           boolean refreshHistory,
           boolean listFiles,
           boolean createDict,
           List<String> subFiles,
           List<String> repositories) throws IndexerException,IOException {

                if (env.getDataRootPath()  == null) {
                    throw new IndexerException("ERROR: Please specify a DATA ROOT path");
                }

                if (env.getSourceRootFile() == null) {
                    File srcConfig = new File(env.getDataRootPath(), "SRC_ROOT");
                    String line = null;
                    if(srcConfig.exists()) {
                        try {
                            BufferedReader sr = new BufferedReader(new FileReader(srcConfig));
                            line = sr.readLine();
                            sr.close();
                        } catch (IOException e) {
                        }
                    }
                    if(line == null) {
                        throw new IndexerException("ERROR: please specify a SRC_ROOT with option -s !");
                    }
                    env.setSourceRoot(line);

                    if (!env.getSourceRootFile().isDirectory()) {
                        throw new IndexerException("ERROR: No such directory:" + line);
                    }
                }

                if (!env.validateExuberantCtags()) {
                    throw new IndexerException("Didn't find Exuberant Ctags");
                }

                if (searchRepositories) {
                    if (env.isVerbose()) {
                        System.out.println("Scanning for repositories...");
                    }
                    env.getRepositories().clear();
                    long start = System.currentTimeMillis();
                    HistoryGuru.getInstance().addRepositories(env.getSourceRootPath());
                    long time = (System.currentTimeMillis() - start) / 1000;
                    if (env.isVerbose()) {
                        System.out.println("Done searching for repositories (" + time + "s)");
                    }
                }

                if (addProjects) {
                    File files[] = env.getSourceRootFile().listFiles();
                    List<Project> projects = env.getProjects();
                    projects.clear();
                    for (File file : files) {
                        if (!file.getName().startsWith(".") && file.isDirectory()) {
                            projects.add(new Project(file.getName(), "/" + file.getName()));
                        }
                    }

                    // The projects should be sorted...
                    Collections.sort(projects, new Comparator<Project>() {
                        public int compare(Project p1, Project p2){
                            String s1 = p1.getDescription();
                            String s2 = p2.getDescription();

                            int ret;
                            if (s1 == null) {
                                ret = (s2 == null) ? 0 : 1;
                            } else {
                                ret = s1.compareTo(s2);
                            }
                            return ret;
                        }
                    });
                }

                if (defaultProject != null) {
                    for (Project p : env.getProjects()) {
                        if (p.getPath().equals(defaultProject)) {
                            env.setDefaultProject(p);
                            break;
                        }
                    }
                }

                if (configFilename != null) {
                    if (env.isVerbose()) {
                        System.out.println("Writing configuration to " + configFilename);
                        System.out.flush();
                    }
                    env.writeConfiguration(new File(configFilename));
                    if (env.isVerbose()) {
                        System.out.println("Done...");
                        System.out.flush();
                    }
                }

                if (refreshHistory) {
                    HistoryGuru.getInstance().createCache();
                } else if (repositories != null && !repositories.isEmpty()) {
                    HistoryGuru.getInstance().createCache(repositories);
                }

                if (listFiles) {
                    IndexDatabase.listAllFiles(subFiles);
                }

                if (createDict) {
                    IndexDatabase.listFrequentTokens(subFiles);
                }
   }


   public void doIndexerExecution(final boolean update, int noThreads, List<String> subFiles,
           IndexChangedListener progress)
      throws IOException {
      RuntimeEnvironment env = RuntimeEnvironment.getInstance();
      env.register();
      log.info("Starting indexExecution");

                ExecutorService executor = Executors.newFixedThreadPool(noThreads);

                if (subFiles == null || subFiles.isEmpty()) {
                    if (update) {
                        IndexDatabase.updateAll(executor, progress);
                    } else if (env.isOptimizeDatabase()) {
                        IndexDatabase.optimizeAll(executor);
                    }
                } else {
                    List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();

                    for (String path : subFiles) {
                        Project project = Project.getProject(path);
                        if (project == null && env.hasProjects()) {
                            System.err.println("Warning: Could not find a project for \"" + path + "\"");
                        } else {
                            IndexDatabase db;
                            if (project == null) {
                                db = new IndexDatabase();
                            } else {
                                db = new IndexDatabase(project);
                            }
                            int idx = dbs.indexOf(db);
                            if (idx != -1) {
                                db = dbs.get(idx);
                            }

                            if (db.addDirectory(path)) {
                                if (idx == -1) {
                                    dbs.add(db);
                                }
                            } else {
                                System.err.println("Warning: Directory does not exist \"" + path + "\"");
                            }
                        }
                    }

                    for (final IndexDatabase db : dbs) {
                            final boolean optimize = env.isOptimizeDatabase();
                            db.addIndexChangedListener(progress);
                            executor.submit(new Runnable() {

                                public void run() {
                                    try {
                                        if (update) {
                                            db.update();
                                        } else if (optimize) {
                                            db.optimize();
                                        }
                                    } catch (Exception e) {
                                        e.printStackTrace();
                                    }
                                }
                            });
                    }
                }

                executor.shutdown();
                while (!executor.isTerminated()) {
                    try {
                        Thread.sleep(1000);
                    } catch (Exception e) {

                    }
                }
   }


   public void sendToConfigHost(RuntimeEnvironment env, String configHost) {
                if (configHost != null) {
                    String[] cfg = configHost.split(":");
                    if (env.isVerbose()) {
                        log.info("Send configuration to: " + configHost);
                    }

                    if (cfg.length == 2) {
                        try {
                            InetAddress host = InetAddress.getByName(cfg[0]);
                            RuntimeEnvironment.getInstance().writeConfiguration(host, Integer.parseInt(cfg[1]));
                        } catch (Exception ex) {
                            log.log(Level.SEVERE,"Failed to send configuration to "
                                    + configHost,ex);
                        }
                    } else {
                        System.err.println("Syntax error: ");
                        for (String s : cfg) {
                            System.err.print("[" + s + "]");
                        }
                        System.err.println();
                    }
                    if (env.isVerbose()) {
                        log.info("Configuration successfully updated");
                    }
                }
    }

    private Indexer() {
    }
}