Indexer.java revision 99
0N/A/*
0N/A * CDDL HEADER START
0N/A *
0N/A * The contents of this file are subject to the terms of the
0N/A * Common Development and Distribution License (the "License").
0N/A * You may not use this file except in compliance with the License.
0N/A *
0N/A * See LICENSE.txt included in this distribution for the specific
0N/A * language governing permissions and limitations under the License.
0N/A *
0N/A * When distributing Covered Code, include this CDDL HEADER in each
0N/A * file and include the License file at LICENSE.txt.
0N/A * If applicable, add the following below this CDDL HEADER, with the
0N/A * fields enclosed by brackets "[]" replaced with your own identifying
0N/A * information: Portions Copyright [yyyy] [name of copyright owner]
0N/A *
0N/A * CDDL HEADER END
0N/A */
0N/A
0N/A/*
930N/A * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
0N/A * Use is subject to license terms.
0N/A */
0N/A
65N/Apackage org.opensolaris.opengrok.index;
125N/A
125N/Aimport java.awt.GraphicsEnvironment;
58N/Aimport java.io.*;
77N/Aimport java.net.InetAddress;
125N/Aimport java.text.ParseException;
125N/Aimport java.util.*;
125N/Aimport org.apache.lucene.document.*;
125N/Aimport org.apache.lucene.index.*;
261N/Aimport org.apache.oro.io.GlobFilenameFilter;
261N/Aimport org.opensolaris.opengrok.analysis.*;
583N/Aimport org.opensolaris.opengrok.configuration.Project;
312N/Aimport org.opensolaris.opengrok.history.ExternalRepository;
312N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
467N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
428N/Aimport org.opensolaris.opengrok.search.scope.MainFrame;
126N/Aimport org.opensolaris.opengrok.util.Getopt;
58N/A
394N/A/**
8N/A * Creates and updates an inverted source index
77N/A * as well as generates Xref, file stats etc., if specified
0N/A * in the options
0N/A */
0N/Apublic class Indexer {
0N/A private static String usage = "Usage: " +
0N/A "opengrok.jar [-qe] [-c ctagsToUse] [-H] [-R filename] [-W filename] [-U hostname:port] [-P] [-p project-path] [-w webapproot] [-i ignore_name [ -i ..]] [-n] [-s SRC_ROOT] DATA_ROOT [subtree .. ]\n" +
0N/A " opengrok.jar [-O | -l | -t] DATA_ROOT\n" +
491N/A "\t-q run quietly\n" +
439N/A "\t-e economical - consumes less disk space\n" +
491N/A "\t-c path to ctags\n" +
465N/A "\t-R Read configuration from file\n" +
465N/A "\t-W Write the current running configuration\n" +
491N/A "\t-U Send configuration to hostname:port\n" +
491N/A "\t-P Generate a project for each toplevel directory\n" +
491N/A "\t-p Use the project specified by the project path as the default project\n" +
886N/A "\t-n Do not generate indexes\n" +
886N/A "\t-H Start a threadpool to read history history\n" +
886N/A "\t-w root URL of the webapp, default is /source\n" +
886N/A "\t-i ignore named files or directories\n" +
886N/A "\t-m Maximum words in a file to index\n" +
886N/A "\t-S Search and add \"External\" repositories (Mercurial etc)\n" +
491N/A "\t-s SRC_ROOT is root directory of source tree\n" +
491N/A "\t default: last used SRC_ROOT\n" +
491N/A "\tDATA_ROOT - is where output of indexer is stored\n" +
491N/A "\tsubtree - only specified files or directories under SRC_ROOT are processed\n" +
65N/A "\t if not specified all files under SRC_ROOT are processed\n" +
65N/A "\n\t-O optimize the index \n" +
65N/A "\t-l list all files in the index \n" +
65N/A "\t-t lists tokens occuring more than 5 times. Useful for building a unix dictionary\n" +
464N/A "\n Eg. java -jar opengrok.jar -s /usr/include /var/tmp/opengrok_data rpc";
0N/A
30N/A private static String options = "qec:R:W:U:Pp:nHw:i:Ss:O:l:t:vD:m:";
58N/A
312N/A /**
312N/A * Program entry point
260N/A * @param argv argument vector
491N/A */
428N/A public static void main(String argv[]) {
376N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
376N/A boolean runIndex = true;
0N/A
11N/A if(argv.length == 0) {
0N/A if (GraphicsEnvironment.isHeadless()) {
240N/A System.err.println("No display available for the Graphical User Interface");
58N/A System.err.println(usage);
58N/A System.exit(1);
58N/A } else {
58N/A MainFrame.main(argv);
77N/A }
207N/A //Run Scope GUI here I am running Indexing GUI for testing
207N/A //new IndexerWizard(null).setVisible(true);
910N/A } else {
491N/A boolean searchRepositories = false;
77N/A ArrayList<String> subFiles = new ArrayList<String>();
260N/A String configFilename = null;
112N/A String configHost = null;
77N/A boolean addProjects = false;
77N/A boolean refreshHistory = false;
77N/A String defaultProject = null;
77N/A
260N/A // Parse command line options:
77N/A Getopt getopt = new Getopt(argv, options);
77N/A try {
77N/A getopt.parse();
491N/A } catch (ParseException ex) {
77N/A System.err.println("OpenGrok: " + ex.getMessage());
491N/A System.err.println(usage);
111N/A System.exit(1);
111N/A }
111N/A
111N/A try{
111N/A int cmd;
111N/A while ((cmd = getopt.getOpt()) != -1) {
111N/A switch (cmd) {
111N/A case 'D' :
491N/A Index.dumpU(getopt.getOptarg());
886N/A System.exit(1);
886N/A break;
886N/A case 'O':
111N/A Index.doOptimize(new File(getopt.getOptarg()));
491N/A System.exit(0);
77N/A break;
77N/A case 'l':
491N/A Index.doList(new File(getopt.getOptarg()));
491N/A System.exit(0);
491N/A break;
491N/A case 't':
77N/A Index.doDict(new File(getopt.getOptarg()));
491N/A System.exit(0);
491N/A break;
491N/A case 'g':
491N/A IgnoredNames.glob = new GlobFilenameFilter(getopt.getOptarg());
491N/A break;
491N/A
491N/A case 'q': env.setVerbose(false); break;
491N/A case 'e': env.setGenerateHtml(false); break;
491N/A case 'P': addProjects = true; break;
491N/A case 'p': defaultProject = getopt.getOptarg(); break;
491N/A case 'c': env.setCtags(getopt.getOptarg()); break;
491N/A case 'w': {
491N/A String webapp = getopt.getOptarg();
491N/A if (webapp.startsWith("/") || webapp.startsWith("http")) {
491N/A ;
491N/A } else {
491N/A webapp = "/" + webapp;
491N/A }
491N/A if (webapp.endsWith("/")) {
491N/A env.setUrlPrefix(webapp + "s?");
491N/A } else {
491N/A env.setUrlPrefix(webapp + "/s?");
491N/A }
491N/A }
491N/A break;
491N/A case 'W': configFilename = getopt.getOptarg(); break;
491N/A case 'U': configHost = getopt.getOptarg(); break;
491N/A case 'R':
491N/A env.readConfiguration(new File(getopt.getOptarg()));
491N/A break;
491N/A case 'n': runIndex = false; break;
491N/A case 'H': refreshHistory = true; break;
491N/A case 'v': env.setVerbose(true); break;
491N/A
491N/A case 's': {
491N/A File file = new File(getopt.getOptarg());
491N/A if (!file.isDirectory()) {
491N/A System.err.println("ERROR: No such directory: " + file.toString());
491N/A System.exit(1);
491N/A }
491N/A
491N/A env.setSourceRoot(file);
491N/A break;
491N/A }
491N/A
491N/A case 'i': IgnoredNames.add(getopt.getOptarg()); break;
799N/A case 'S' : searchRepositories = true; break;
799N/A case 'm' : {
799N/A try {
886N/A env.setIndexWordLimit(Integer.parseInt(getopt.getOptarg()));
886N/A } catch (NumberFormatException exp) {
886N/A System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());
886N/A System.exit(1);
886N/A }
891N/A break;
886N/A }
891N/A default:
886N/A System.err.println("Unknown option: " + (char)cmd);
886N/A System.exit(1);
886N/A }
886N/A }
886N/A
886N/A int optind = getopt.getOptind();
491N/A if (optind != -1) {
491N/A if (optind < argv.length) {
491N/A env.setDataRoot(argv[optind]);
491N/A ++optind;
491N/A }
491N/A
491N/A while (optind < argv.length) {
491N/A subFiles.add(argv[optind]);
491N/A ++optind;
491N/A }
491N/A }
491N/A
491N/A if (env.getDataRootPath() == null) {
491N/A System.err.println("ERROR: Please specify a DATA ROOT path");
491N/A System.err.println(usage);
491N/A System.exit(1);
491N/A }
491N/A
491N/A if (env.getSourceRootFile() == null) {
491N/A File srcConfig = new File(env.getDataRootPath(), "SRC_ROOT");
491N/A String line = null;
491N/A if(srcConfig.exists()) {
491N/A try {
491N/A BufferedReader sr = new BufferedReader(new FileReader(srcConfig));
491N/A line = sr.readLine();
491N/A sr.close();
491N/A } catch (IOException e) {
491N/A }
491N/A }
491N/A if(line == null) {
491N/A System.err.println("ERROR: please specify a SRC_ROOT with option -s !");
491N/A System.err.println(usage);
491N/A System.exit(1);
77N/A }
491N/A env.setSourceRoot(line);
491N/A
490N/A if (!env.getSourceRootFile().isDirectory()) {
490N/A System.err.println("ERROR: No such directory:" + line);
490N/A System.err.println(usage);
490N/A System.exit(1);
490N/A }
490N/A }
489N/A
490N/A if (!env.validateExuberantCtags()) {
491N/A System.exit(1);
491N/A }
490N/A
490N/A if (searchRepositories) {
490N/A if (env.isVerbose()) {
490N/A System.out.println("Scanning for repositories...");
490N/A }
490N/A env.getRepositories().clear();
490N/A long start = System.currentTimeMillis();
490N/A HistoryGuru.getInstance().addExternalRepositories(env.getSourceRootPath());
491N/A long time = (System.currentTimeMillis() - start) / 1000;
491N/A if (env.isVerbose()) {
491N/A System.out.println("Done searching for repositories (" + time + "s)");
491N/A }
491N/A }
491N/A
491N/A if (addProjects) {
491N/A File files[] = env.getSourceRootFile().listFiles();
491N/A List<Project> projects = env.getProjects();
491N/A projects.clear();
491N/A for (File file : files) {
491N/A if (!file.getName().startsWith(".") && file.isDirectory()) {
491N/A projects.add(new Project(file.getName(), "/" + file.getName()));
491N/A }
491N/A }
491N/A
491N/A // The projects should be sorted...
491N/A Collections.sort(projects, new Comparator<Project>() {
491N/A public int compare(Project p1, Project p2){
491N/A String s1 = p1.getDescription();
491N/A String s2 = p2.getDescription();
491N/A
491N/A int ret;
491N/A if (s1 == null) {
491N/A ret = (s2 == null) ? 0 : 1;
491N/A } else {
99N/A ret = s1.compareTo(s2);
491N/A }
491N/A return ret;
491N/A }
491N/A });
491N/A }
491N/A
491N/A if (defaultProject != null) {
491N/A for (Project p : env.getProjects()) {
491N/A if (p.getPath().equals(defaultProject)) {
126N/A env.setDefaultProject(p);
126N/A break;
126N/A }
491N/A }
491N/A }
491N/A
491N/A if (configFilename != null) {
491N/A if (env.isVerbose()) {
491N/A System.out.println("Writing configuration to " + configFilename);
491N/A System.out.flush();
491N/A }
491N/A env.writeConfiguration(new File(configFilename));
491N/A if (env.isVerbose()) {
491N/A System.out.println("Done...");
491N/A System.out.flush();
491N/A }
126N/A }
491N/A
491N/A if (refreshHistory) {
491N/A for (Map.Entry<String, ExternalRepository> entry : RuntimeEnvironment.getInstance().getRepositories().entrySet()) {
491N/A try {
491N/A entry.getValue().createCache();
491N/A } catch (Exception e) {
491N/A System.err.println("Failed to generate history cache.");
491N/A e.printStackTrace();
491N/A }
491N/A }
491N/A }
491N/A
491N/A if (runIndex) {
491N/A Index idx = new Index(env.isVerbose() ? new StandardPrinter(System.out) : new NullPrinter(), new StandardPrinter(System.err));
491N/A idx.runIndexer(env.getDataRootFile(), env.getSourceRootFile(), subFiles, env.isGenerateHtml());
491N/A }
491N/A
491N/A if (configHost != null) {
491N/A String[] cfg = configHost.split(":");
491N/A if (env.isVerbose()) {
491N/A System.out.println("Send configuration to: " + configHost);
491N/A }
126N/A
491N/A if (cfg.length == 2) {
491N/A try {
491N/A InetAddress host = InetAddress.getByName(cfg[0]);
126N/A RuntimeEnvironment.getInstance().writeConfiguration(host, Integer.parseInt(cfg[1]));
126N/A } catch (Exception ex) {
491N/A System.err.println("Failed to send configuration to " + configHost);
491N/A ex.printStackTrace();
491N/A }
491N/A } else {
491N/A System.err.println("Syntax error: ");
491N/A for (String s : cfg) {
491N/A System.err.print("[" + s + "]");
491N/A }
491N/A System.err.println();
491N/A }
491N/A if (env.isVerbose()) {
491N/A System.out.println("Configuration successfully updated");
491N/A }
491N/A }
491N/A } catch (Exception e) {
491N/A System.err.println("Error: [ main ] " + e);
491N/A if (env.isVerbose()) e.printStackTrace();
491N/A System.exit(1);
491N/A }
491N/A }
491N/A }
491N/A}
491N/A