Indexer.java revision 1463
0N/A/*
0N/A * CDDL HEADER START
0N/A *
0N/A * The contents of this file are subject to the terms of the
0N/A * Common Development and Distribution License (the "License").
0N/A * You may not use this file except in compliance with the License.
0N/A *
0N/A * See LICENSE.txt included in this distribution for the specific
0N/A * language governing permissions and limitations under the License.
0N/A *
0N/A * When distributing Covered Code, include this CDDL HEADER in each
0N/A * file and include the License file at LICENSE.txt.
0N/A * If applicable, add the following below this CDDL HEADER, with the
0N/A * fields enclosed by brackets "[]" replaced with your own identifying
0N/A * information: Portions Copyright [yyyy] [name of copyright owner]
0N/A *
0N/A * CDDL HEADER END
0N/A */
0N/A
0N/A/*
1393N/A * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
1190N/A *
1185N/A * Portions Copyright 2011 Jens Elkner.
0N/A */
0N/Apackage org.opensolaris.opengrok.index;
65N/A
125N/Aimport java.io.File;
125N/Aimport java.io.IOException;
1185N/Aimport java.lang.reflect.Field;
58N/Aimport java.net.InetAddress;
77N/Aimport java.text.ParseException;
125N/Aimport java.util.ArrayList;
125N/Aimport java.util.Collections;
125N/Aimport java.util.Comparator;
1092N/Aimport java.util.HashMap;
1016N/Aimport java.util.HashSet;
125N/Aimport java.util.List;
1092N/Aimport java.util.Map;
261N/Aimport java.util.concurrent.ExecutorService;
261N/Aimport java.util.concurrent.Executors;
583N/Aimport java.util.concurrent.TimeUnit;
312N/Aimport java.util.logging.Level;
1062N/Aimport java.util.logging.LogManager;
312N/Aimport java.util.logging.Logger;
1264N/A
467N/Aimport org.opensolaris.opengrok.Info;
428N/Aimport org.opensolaris.opengrok.OpenGrokLogger;
126N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
1088N/Aimport org.opensolaris.opengrok.configuration.Configuration;
58N/Aimport org.opensolaris.opengrok.configuration.Project;
394N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
1185N/Aimport org.opensolaris.opengrok.history.HistoryException;
8N/Aimport org.opensolaris.opengrok.history.HistoryGuru;
1185N/Aimport org.opensolaris.opengrok.history.Repository;
1185N/Aimport org.opensolaris.opengrok.history.RepositoryFactory;
1185N/Aimport org.opensolaris.opengrok.history.RepositoryInfo;
1264N/Aimport org.opensolaris.opengrok.util.Executor;
77N/Aimport org.opensolaris.opengrok.util.Getopt;
1419N/Aimport org.opensolaris.opengrok.web.Prefix;
0N/A
0N/A/**
0N/A * Creates and updates an inverted source index
0N/A * as well as generates Xref, file stats etc., if specified
0N/A * in the options
0N/A */
491N/A@SuppressWarnings({"PMD.AvoidPrintStackTrace", "PMD.SystemPrintln"})
439N/Apublic final class Indexer {
1238N/A private static final String ON = "on";
1238N/A private static final String OFF = "off";
491N/A private static Indexer index = new Indexer();
1463N/A static final Logger log = Logger.getLogger(Indexer.class.getName());
491N/A
886N/A private static final String DERBY_EMBEDDED_DRIVER =
886N/A "org.apache.derby.jdbc.EmbeddedDriver";
886N/A
886N/A private static final String DERBY_CLIENT_DRIVER =
886N/A "org.apache.derby.jdbc.ClientDriver";
886N/A
1463N/A /**
1463N/A * Get the indexer instance.
1463N/A * @return a singleton
1463N/A */
491N/A public static Indexer getInstance() {
491N/A return index;
491N/A }
491N/A
65N/A /**
65N/A * Program entry point
65N/A * @param argv argument vector
65N/A */
464N/A @SuppressWarnings("PMD.UseStringBufferForStringAppends")
0N/A public static void main(String argv[]) {
58N/A boolean runIndex = true;
312N/A boolean update = true;
312N/A boolean optimizedChanged = false;
1185N/A ArrayList<String> zapCache = new ArrayList<String>();
260N/A CommandLineOptions cmdOptions = new CommandLineOptions();
491N/A
428N/A if (argv.length == 0) {
376N/A System.err.println(cmdOptions.getUsage());
376N/A System.exit(1);
0N/A } else {
1264N/A Executor.registerErrorHandler();
11N/A boolean searchRepositories = false;
0N/A ArrayList<String> subFiles = new ArrayList<String>();
240N/A ArrayList<String> repositories = new ArrayList<String>();
1016N/A HashSet<String> allowedSymlinks = new HashSet<String>();
58N/A String configFilename = null;
58N/A String configHost = null;
58N/A boolean addProjects = false;
58N/A boolean refreshHistory = false;
77N/A String defaultProject = null;
207N/A boolean listFiles = false;
1185N/A boolean listRepos = false;
1463N/A boolean listTokens = false;
910N/A int noThreads = 2 + (2 * Runtime.getRuntime().availableProcessors());
1190N/A
77N/A // Parse command line options:
260N/A Getopt getopt = new Getopt(argv, cmdOptions.getCommandString());
112N/A
77N/A try {
77N/A getopt.parse();
77N/A } catch (ParseException ex) {
1327N/A System.err.println(ex.getMessage());
260N/A System.err.println(cmdOptions.getUsage());
77N/A System.exit(1);
77N/A }
77N/A
491N/A try {
1088N/A Configuration cfg = null;
77N/A int cmd;
491N/A
111N/A // We need to read the configuration file first, since we
111N/A // will try to overwrite options..
111N/A while ((cmd = getopt.getOpt()) != -1) {
111N/A if (cmd == 'R') {
1088N/A cfg = Configuration.read(new File(getopt.getOptarg()));
111N/A break;
111N/A }
111N/A }
491N/A
1088N/A if (cfg == null) {
1088N/A cfg = new Configuration();
1088N/A }
1088N/A
1088N/A String databaseDriver = cfg.getDatabaseDriver();
1088N/A String databaseURL = cfg.getDatabaseUrl();
886N/A
111N/A // Now we can handle all the other options..
491N/A getopt.reset();
77N/A while ((cmd = getopt.getOpt()) != -1) {
77N/A switch (cmd) {
491N/A case 'q':
1088N/A cfg.setVerbose(false);
1062N/A OpenGrokLogger.setOGConsoleLogLevel(Level.WARNING);
491N/A break;
491N/A case 'e':
1088N/A cfg.setGenerateHtml(false);
491N/A break;
491N/A case 'P':
491N/A addProjects = true;
491N/A break;
491N/A case 'p':
491N/A defaultProject = getopt.getOptarg();
491N/A break;
491N/A case 'c':
1088N/A cfg.setCtags(getopt.getOptarg());
491N/A break;
491N/A case 'w':
491N/A {
491N/A String webapp = getopt.getOptarg();
491N/A if (webapp.charAt(0) != '/' && !webapp.startsWith("http")) {
491N/A webapp = "/" + webapp;
491N/A }
491N/A if (webapp.endsWith("/")) {
1419N/A webapp = webapp.substring(0, webapp.length()-1);
491N/A }
1419N/A cfg.setUrlPrefix(webapp + Prefix.SEARCH_R + '?');
491N/A }
491N/A break;
491N/A case 'W':
491N/A configFilename = getopt.getOptarg();
491N/A break;
491N/A case 'U':
491N/A configHost = getopt.getOptarg();
491N/A break;
491N/A case 'R':
491N/A // already handled
491N/A break;
1016N/A case 'N':
1016N/A allowedSymlinks.add(getopt.getOptarg());
1016N/A break;
491N/A case 'n':
491N/A runIndex = false;
491N/A break;
491N/A case 'H':
491N/A refreshHistory = true;
491N/A break;
491N/A case 'h':
491N/A repositories.add(getopt.getOptarg());
491N/A break;
799N/A case 'D':
1088N/A cfg.setHistoryCacheInDB(true);
799N/A break;
886N/A case 'j':
886N/A databaseDriver = getopt.getOptarg();
886N/A // Should be a full class name, but we also accept
886N/A // the shorthands "client" and "embedded". Expand
886N/A // the shorthands here.
891N/A if ("client".equals(databaseDriver)) {
886N/A databaseDriver = DERBY_CLIENT_DRIVER;
891N/A } else if ("embedded".equals(databaseDriver)) {
886N/A databaseDriver = DERBY_EMBEDDED_DRIVER;
886N/A }
886N/A break;
886N/A case 'u':
886N/A databaseURL = getopt.getOptarg();
886N/A break;
491N/A case 'r':
491N/A {
491N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {
1088N/A cfg.setRemoteScmSupported(true);
491N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
1088N/A cfg.setRemoteScmSupported(false);
491N/A } else {
491N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -r");
491N/A System.err.println(" Ex: \"-r on\" will allow retrival for remote SCM systems");
491N/A System.err.println(" \"-r off\" will ignore SCM for remote systems");
491N/A }
491N/A }
491N/A break;
1393N/A case 'o':
1416N/A File ctagsOptions = new File(getopt.getOptarg());
1416N/A if (!(ctagsOptions.isFile() && ctagsOptions.canRead()))
1416N/A {
1416N/A System.err.println("ERROR: File '" + ctagsOptions
1416N/A + "' not found for the -o option");
1393N/A System.exit(1);
1393N/A }
1416N/A cfg.setCtagsOptionsFile(ctagsOptions.getPath());
1393N/A break;
491N/A case 'O':
491N/A {
1088N/A boolean oldval = cfg.isOptimizeDatabase();
491N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {
1088N/A cfg.setOptimizeDatabase(true);
491N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
1088N/A cfg.setOptimizeDatabase(false);
491N/A } else {
491N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -O");
491N/A System.err.println(" Ex: \"-O on\" will optimize the database as part of the index generation");
491N/A System.err.println(" \"-O off\" disable optimization of the index database");
491N/A }
1088N/A if (oldval != cfg.isOptimizeDatabase()) {
491N/A optimizedChanged = true;
491N/A }
491N/A }
491N/A break;
491N/A case 'v':
1088N/A cfg.setVerbose(true);
1062N/A OpenGrokLogger.setOGConsoleLogLevel(Level.INFO);
491N/A break;
1115N/A case 'C':
1115N/A cfg.setPrintProgress(true);
1115N/A break;
77N/A
491N/A case 's':
1106N/A {
1106N/A File sourceRoot = new File(getopt.getOptarg());
1106N/A if (!sourceRoot.isDirectory()) {
1106N/A System.err.println("ERROR: Source root must be a directory");
1106N/A System.exit(1);
489N/A }
1106N/A cfg.setSourceRoot(sourceRoot.getCanonicalPath());
490N/A break;
1106N/A }
491N/A case 'd':
1106N/A {
1106N/A File dataRoot = new File(getopt.getOptarg());
1106N/A if (!dataRoot.exists() && !dataRoot.mkdirs()) {
1106N/A System.err.println("ERROR: Cannot create data root");
1106N/A System.exit(1);
490N/A }
1106N/A if (!dataRoot.isDirectory()) {
1106N/A System.err.println("ERROR: Data root must be a directory");
1106N/A System.exit(1);
1106N/A }
1106N/A cfg.setDataRoot(dataRoot.getCanonicalPath());
490N/A break;
1106N/A }
491N/A case 'i':
1088N/A cfg.getIgnoredNames().add(getopt.getOptarg());
491N/A break;
1026N/A case 'I':
1088N/A cfg.getIncludedNames().add(getopt.getOptarg());
1026N/A break;
491N/A case 'S':
491N/A searchRepositories = true;
491N/A break;
491N/A case 'Q':
491N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {
1088N/A cfg.setQuickContextScan(true);
491N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
1088N/A cfg.setQuickContextScan(false);
491N/A } else {
491N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -Q");
491N/A System.err.println(" Ex: \"-Q on\" will just scan a \"chunk\" of the file and insert \"[..all..]\"");
491N/A System.err.println(" \"-Q off\" will try to build a more accurate list by reading the complete file.");
491N/A }
491N/A
491N/A break;
491N/A case 'm': {
491N/A try {
1088N/A cfg.setIndexWordLimit(Integer.parseInt(getopt.getOptarg()));
491N/A } catch (NumberFormatException exp) {
491N/A System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());
491N/A System.exit(1);
491N/A }
491N/A break;
99N/A }
491N/A case 'a':
491N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {
1088N/A cfg.setAllowLeadingWildcard(true);
491N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
1088N/A cfg.setAllowLeadingWildcard(false);
491N/A } else {
491N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -a");
491N/A System.err.println(" Ex: \"-a on\" will allow a search to start with a wildcard");
491N/A System.err.println(" \"-a off\" will disallow a search to start with a wildcard");
126N/A System.exit(1);
126N/A }
126N/A
491N/A break;
491N/A
491N/A case 'A':
491N/A {
491N/A String[] arg = getopt.getOptarg().split(":");
491N/A if (arg.length != 2) {
491N/A System.err.println("ERROR: You must specify: -A extension:class");
491N/A System.err.println(" Ex: -A foo:org.opensolaris.opengrok.analysis.c.CAnalyzer");
491N/A System.err.println(" will use the C analyzer for all files ending with .foo");
491N/A System.err.println(" Ex: -A c:-");
491N/A System.err.println(" will disable the c-analyzer for for all files ending with .c");
491N/A System.exit(1);
491N/A }
126N/A
491N/A arg[0] = arg[0].substring(arg[0].lastIndexOf('.') + 1).toUpperCase();
491N/A if (arg[1].equals("-")) {
491N/A AnalyzerGuru.addExtension(arg[0], null);
491N/A break;
491N/A }
491N/A
491N/A try {
1327N/A AnalyzerGuru.addExtension(arg[0],
1327N/A AnalyzerGuru.findFactory(arg[1]));
491N/A } catch (Exception e) {
1054N/A log.log(Level.SEVERE, "Unable to use {0} as a FileAnalyzerFactory", arg[1]);
1327N/A log.log(Level.FINE, "main", e);
491N/A System.exit(1);
491N/A }
491N/A }
491N/A break;
491N/A case 'L':
1088N/A cfg.setWebappLAF(getopt.getOptarg());
491N/A break;
491N/A case 'T':
126N/A try {
491N/A noThreads = Integer.parseInt(getopt.getOptarg());
491N/A } catch (NumberFormatException exp) {
491N/A System.err.println("ERROR: Failed to parse argument to \"-T\": " + exp.getMessage());
126N/A System.exit(1);
126N/A }
491N/A break;
994N/A case 'z':
994N/A try {
1088N/A cfg.setScanningDepth(Integer.parseInt(getopt.getOptarg()));
994N/A } catch (NumberFormatException exp) {
994N/A System.err.println("ERROR: Failed to parse argument to \"-z\": " + exp.getMessage());
994N/A System.exit(1);
994N/A }
994N/A break;
491N/A case 'l':
491N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {
1088N/A cfg.setUsingLuceneLocking(true);
491N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
1088N/A cfg.setUsingLuceneLocking(false);
491N/A } else {
491N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -l");
491N/A System.err.println(" Ex: \"-l on\" will enable locks in Lucene");
491N/A System.err.println(" \"-l off\" will disable locks in Lucene");
491N/A }
491N/A break;
1100N/A case 'B':
1100N/A cfg.setUserPage(getopt.getOptarg());
1100N/A break;
1100N/A case 'X':
1100N/A cfg.setUserPageSuffix(getopt.getOptarg());
1100N/A break;
491N/A case 'V':
491N/A System.out.println(Info.getFullVersion());
491N/A System.exit(0);
491N/A break;
1185N/A case 'k':
1185N/A zapCache.add(getopt.getOptarg());
1185N/A break;
1185N/A case 'K':
1185N/A listRepos = true;
1185N/A break;
491N/A case '?':
491N/A System.err.println(cmdOptions.getUsage());
491N/A System.exit(0);
491N/A break;
1185N/A case 't':
1185N/A try {
1185N/A int tmp = Integer.parseInt(getopt.getOptarg());
1185N/A cfg.setTabSize(tmp);
1185N/A } catch (NumberFormatException exp) {
1185N/A System.err.println("ERROR: Failed to parse argument to \"-t\": " + exp.getMessage());
1185N/A System.exit(1);
1185N/A }
1190N/A break;
1330N/A case 'Z':
1330N/A cfg.setChattyStatusPage(true);
1330N/A break;
1348N/A case 'E':
1348N/A cfg.setCompressXref(false);
1348N/A break;
491N/A default:
491N/A System.err.println("Internal Error - Unimplemented cmdline option: " + (char) cmd);
261N/A System.exit(1);
0N/A }
0N/A }
1463N/A String s = System.getProperty(Configuration.DIRLIST_TODAY_PROPERTY_KEY);
1463N/A if (s != null) {
1463N/A cfg.setDirlistUseToday(Boolean.parseBoolean(s));
1463N/A }
1463N/A s = System.getProperty(Configuration.DIRLIST_DATE_PROPERTY_KEY);
1463N/A if (s != null) {
1463N/A cfg.setDirlistDatePattern(s);
1463N/A }
1190N/A List<Class<? extends Repository>> repositoryClasses =
1185N/A RepositoryFactory.getRepositoryClasses();
1185N/A for (Class<? extends Repository> clazz : repositoryClasses) {
1185N/A try {
1185N/A Field f = clazz.getDeclaredField("CMD_PROPERTY_KEY");
1185N/A Object key = f.get(null);
1185N/A if (key != null) {
1190N/A cfg.setRepoCmd(clazz.getCanonicalName(),
1185N/A System.getProperty(key.toString()));
1185N/A }
1185N/A } catch (Exception e) {
1185N/A // don't care
1185N/A }
1185N/A }
77N/A int optind = getopt.getOptind();
491N/A if (optind != -1) {
77N/A while (optind < argv.length) {
77N/A subFiles.add(argv[optind]);
77N/A ++optind;
77N/A }
77N/A }
1190N/A
1062N/A //logging starts here
1088N/A if (cfg.isVerbose()) {
1062N/A String fn=LogManager.getLogManager().getProperty("java.util.logging.FileHandler.pattern");
1135N/A if (fn!=null) {System.out.println("Logging filehandler pattern: "+fn);}
1062N/A }
1190N/A
1088N/A if (cfg.isHistoryCacheInDB()) {
886N/A // The default database driver is Derby's client driver.
886N/A if (databaseDriver == null) {
886N/A databaseDriver = DERBY_CLIENT_DRIVER;
886N/A }
886N/A
886N/A // The default URL depends on the database driver.
886N/A if (databaseURL == null) {
886N/A StringBuilder defaultURL = new StringBuilder();
886N/A defaultURL.append("jdbc:derby:");
886N/A if (databaseDriver.equals(DERBY_EMBEDDED_DRIVER)) {
1088N/A defaultURL.append(cfg.getDataRoot())
886N/A .append(File.separator);
886N/A } else {
886N/A defaultURL.append("//localhost/");
886N/A }
886N/A defaultURL.append("cachedb;create=true");
886N/A databaseURL = defaultURL.toString();
886N/A }
886N/A }
886N/A
1088N/A cfg.setDatabaseDriver(databaseDriver);
1088N/A cfg.setDatabaseUrl(databaseURL);
886N/A
1017N/A // automatically allow symlinks that are directly in source root
1088N/A String file = cfg.getSourceRoot();
1090N/A if (file != null) {
1090N/A File sourceRootFile = new File(file);
1017N/A File[] projectDirs = sourceRootFile.listFiles();
1017N/A if (projectDirs != null) {
1017N/A for (File projectDir : projectDirs) {
1017N/A if (!projectDir.getCanonicalPath().equals(projectDir.getAbsolutePath())) {
1017N/A allowedSymlinks.add(projectDir.getAbsolutePath());
1017N/A }
1017N/A }
1017N/A }
1017N/A }
1190N/A
1088N/A allowedSymlinks.addAll(cfg.getAllowedSymlinks());
1088N/A cfg.setAllowedSymlinks(allowedSymlinks);
1088N/A
1088N/A //Set updated configuration in RuntimeEnvironment
1088N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
1088N/A env.setConfiguration(cfg);
1016N/A
1463N/A Indexer.prepareIndexer(env, searchRepositories, addProjects,
491N/A defaultProject, configFilename, refreshHistory,
1463N/A listFiles, listTokens, subFiles, repositories,
1185N/A zapCache, listRepos);
1185N/A if (listRepos || !zapCache.isEmpty()) {
1185N/A return;
1185N/A }
491N/A if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
491N/A IndexChangedListener progress = new DefaultIndexChangedListener();
1463N/A Indexer.doIndexerExecution(update, noThreads, subFiles,
491N/A progress);
491N/A }
1463N/A Indexer.sendToConfigHost(configHost);
491N/A } catch (IndexerException ex) {
1327N/A log.log(Level.SEVERE, "Exception running indexer: " + ex.getMessage());
1327N/A log.log(Level.FINE, "main", ex);
491N/A System.err.println(cmdOptions.getUsage());
491N/A System.exit(1);
1264N/A } catch (Throwable e) {
1327N/A log.log(Level.SEVERE, e.getLocalizedMessage());
1327N/A log.log(Level.FINE, "main", e);
491N/A System.exit(1);
312N/A }
491N/A }
491N/A
491N/A }
491N/A
1093N/A // PMD wants us to use length() > 0 && charAt(0) instead of startsWith()
1093N/A // for performance. We prefer clarity over performance here, so silence it.
1463N/A /**
1463N/A * Prepare the environment to execute indexing. Basically it does some
1463N/A * consistency checks wrt. commadline options sets and executes simple tasks
1463N/A * like zapping or refreshing the history cache, list available projects
1463N/A * including their path, create spelling index for related projects,
1463N/A * writing the config file.
1463N/A *
1463N/A * @param env Environment to use/check/update.
1463N/A * @param searchRepositories If {@code true} scan for new repositories in
1463N/A * the source root directory.
1463N/A * @param addProjects If {@code true} add automatically projects found
1463N/A * @param defaultProject The name of the project to use as default. Might
1463N/A * be {@code null}.
1463N/A * @param configFilename The pathname of the configuration file to write.
1463N/A * If {@code null} writing the config file gets skipped.
1463N/A * @param refreshHistory If {@code true}, create/update the history cache
1463N/A * for the given or all projects.
1463N/A * @param listFiles If {@code true}, print a list of all files of
1463N/A * the index database for given projects to stdout.
1463N/A * @param listTokens If {@code true}, print a list of frequent
1463N/A * tokens for related projects to stdout.
1463N/A * @param subFiles Determine target projects by the given repo
1463N/A * files when <var>listFiles</var> or <var>createDict</var> is set to
1463N/A * {@code true}. {@code null} implies "all projects".
1463N/A * @param repositories Target repositories.{@code null} implies
1463N/A * "all projects".
1463N/A * @param zapCache List of project names, whose history cache should
1463N/A * be zapped. An Asterisk ('*') implies zap all. Ignored if empty.
1463N/A * @param listRepoPathes If {@code true}, just list available projects
1463N/A * incl. their path, zap projects (if set) and return.
1463N/A * @throws IndexerException
1463N/A * @throws IOException
1463N/A * @throws NullPointerException if <var>env</var> or <var>zapCache</var> is
1463N/A * {@code null}
1463N/A */
1093N/A @SuppressWarnings("PMD.SimplifyStartsWith")
1463N/A public static void prepareIndexer(RuntimeEnvironment env,
491N/A boolean searchRepositories,
491N/A boolean addProjects,
491N/A String defaultProject,
491N/A String configFilename,
491N/A boolean refreshHistory,
491N/A boolean listFiles,
1463N/A boolean listTokens,
491N/A List<String> subFiles,
1185N/A List<String> repositories,
1185N/A List<String> zapCache,
1185N/A boolean listRepoPathes) throws IndexerException, IOException {
491N/A
491N/A if (env.getDataRootPath() == null) {
1327N/A throw new IndexerException("Please specify a DATA ROOT path");
491N/A }
491N/A
491N/A if (env.getSourceRootFile() == null) {
1327N/A throw new IndexerException("Please specify a SRC_ROOT with option -s");
491N/A }
491N/A
491N/A if (!env.validateExuberantCtags()) {
491N/A throw new IndexerException("Didn't find Exuberant Ctags");
491N/A }
1185N/A if (zapCache == null) {
1214N/A throw new IndexerException("Internal error, zapCache shouldn't be null");
1185N/A }
491N/A
1190N/A if (searchRepositories || listRepoPathes || !zapCache.isEmpty()) {
1054N/A log.log(Level.INFO,"Scanning for repositories...");
491N/A long start = System.currentTimeMillis();
491N/A HistoryGuru.getInstance().addRepositories(env.getSourceRootPath());
1190N/A long time = (System.currentTimeMillis() - start) / 1000;
1463N/A log.log(Level.INFO, "Done ({0}s)", Long.valueOf(time));
1185N/A if (listRepoPathes || !zapCache.isEmpty()) {
1185N/A List<RepositoryInfo> repos = env.getRepositories();
1185N/A String prefix = env.getSourceRootPath();
1185N/A if (listRepoPathes) {
1185N/A if (repos.isEmpty()) {
1185N/A System.out.println("No repositories found.");
1185N/A return;
1185N/A }
1185N/A System.out.println("Repositories in " + prefix + ":");
1185N/A for (RepositoryInfo info : env.getRepositories()) {
1185N/A String dir = info.getDirectoryName();
1185N/A System.out.println(dir.substring(prefix.length()));
1185N/A }
1185N/A }
1185N/A if (!zapCache.isEmpty()) {
1185N/A HashSet<String> toZap = new HashSet<String>(zapCache.size() << 1);
1185N/A boolean all = false;
1185N/A for (String repo : zapCache) {
1236N/A if ("*".equals(repo)) {
1185N/A all = true;
1185N/A break;
1185N/A }
1185N/A if (repo.startsWith(prefix)) {
1185N/A repo = repo.substring(prefix.length());
1185N/A }
1185N/A toZap.add(repo);
1185N/A }
1185N/A if (all) {
1185N/A toZap.clear();
1185N/A for (RepositoryInfo info : env.getRepositories()) {
1185N/A toZap.add(info.getDirectoryName()
1190N/A .substring(prefix.length()));
1190N/A }
1185N/A }
1185N/A try {
1185N/A HistoryGuru.getInstance().removeCache(toZap);
1185N/A } catch (HistoryException e) {
1185N/A log.warning("Clearing history cache faild: "
1185N/A + e.getLocalizedMessage());
1185N/A }
1185N/A }
1185N/A return;
1185N/A }
491N/A }
491N/A
491N/A if (addProjects) {
491N/A File files[] = env.getSourceRootFile().listFiles();
491N/A List<Project> projects = env.getProjects();
1092N/A
1092N/A // Keep a copy of the old project list so that we can preserve
1092N/A // the customization of existing projects.
1092N/A Map<String, Project> oldProjects = new HashMap<String, Project>();
1092N/A for (Project p : projects) {
1092N/A oldProjects.put(p.getPath(), p);
1092N/A }
1092N/A
491N/A projects.clear();
1092N/A
1092N/A // Add a project for each top-level directory in source root.
491N/A for (File file : files) {
1092N/A String name = file.getName();
1092N/A String path = "/" + name;
1092N/A if (oldProjects.containsKey(path)) {
1092N/A // This is an existing object. Reuse the old project,
1092N/A // possibly with customizations, instead of creating a
1092N/A // new with default values.
1092N/A projects.add(oldProjects.get(path));
1092N/A } else if (!name.startsWith(".") && file.isDirectory()) {
1092N/A // Found a new directory with no matching project, so
1092N/A // create a new project with default properties.
491N/A Project p = new Project();
491N/A p.setDescription(name);
1092N/A p.setPath(path);
1185N/A p.setTabSize(env.getConfiguration().getTabSize());
491N/A projects.add(p);
491N/A }
491N/A }
491N/A
491N/A // The projects should be sorted...
491N/A Collections.sort(projects, new Comparator<Project>() {
491N/A
1054N/A @Override
491N/A public int compare(Project p1, Project p2) {
491N/A String s1 = p1.getDescription();
491N/A String s2 = p2.getDescription();
312N/A
491N/A int ret;
491N/A if (s1 == null) {
491N/A ret = (s2 == null) ? 0 : 1;
491N/A } else {
491N/A ret = s1.compareTo(s2);
491N/A }
491N/A return ret;
0N/A }
491N/A });
491N/A }
77N/A
491N/A if (defaultProject != null) {
491N/A for (Project p : env.getProjects()) {
491N/A if (p.getPath().equals(defaultProject)) {
491N/A env.setDefaultProject(p);
491N/A break;
0N/A }
491N/A }
491N/A }
77N/A
1190N/A if (configFilename != null) {
1327N/A log.log(Level.INFO, "Writing configuration to ''{0}''", configFilename);
1190N/A env.writeConfiguration(new File(configFilename));
1327N/A log.info("Done.");
491N/A }
491N/A
491N/A if (refreshHistory) {
1062N/A log.log(Level.INFO, "Generating history cache for all repositories ...");
491N/A HistoryGuru.getInstance().createCache();
1327N/A log.info("Done.");
491N/A } else if (repositories != null && !repositories.isEmpty()) {
1062N/A log.log(Level.INFO, "Generating history cache for specified repositories ...");
491N/A HistoryGuru.getInstance().createCache(repositories);
1327N/A log.info("Done.");
491N/A }
491N/A
491N/A if (listFiles) {
491N/A IndexDatabase.listAllFiles(subFiles);
491N/A }
491N/A
1463N/A if (listTokens) {
491N/A IndexDatabase.listFrequentTokens(subFiles);
491N/A }
491N/A }
491N/A
1463N/A /**
1463N/A * Index and optionally optimize related databases. Obtains an
1463N/A * {@link RuntimeEnvironment} instance and registers with it, so that
1463N/A * required infos are obtained from there.
1463N/A *
1463N/A * @param update If {@code true} update the index for related projects.
1463N/A * @param noThreads Number of threads to use for indexing.
1463N/A * @param subFiles Use the giben repo files to determine the projects,
1463N/A * which should be indexed or index db optimized.
1463N/A * @param progress If {@code true} show progress when indexing. Wrt.
1463N/A * to logging and terminal flooding a really bad idea.
1463N/A * @throws IOException
1463N/A * @see IndexDatabase#update(ExecutorService, IndexChangedListener, List)
1463N/A */
1463N/A public static void doIndexerExecution(final boolean update, int noThreads,
1463N/A List<String> subFiles, IndexChangedListener progress)
1463N/A throws IOException
1463N/A {
1185N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance().register();
1329N/A log.info("Start indexing ...");
491N/A
491N/A ExecutorService executor = Executors.newFixedThreadPool(noThreads);
491N/A
491N/A if (subFiles == null || subFiles.isEmpty()) {
491N/A if (update) {
491N/A IndexDatabase.updateAll(executor, progress);
491N/A } else if (env.isOptimizeDatabase()) {
491N/A IndexDatabase.optimizeAll(executor);
491N/A }
491N/A } else {
491N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
491N/A
491N/A for (String path : subFiles) {
491N/A Project project = Project.getProject(path);
491N/A if (project == null && env.hasProjects()) {
1327N/A log.warning("Could not find a project for '" + path + "'");
491N/A } else {
491N/A IndexDatabase db;
491N/A if (project == null) {
491N/A db = new IndexDatabase();
491N/A } else {
491N/A db = new IndexDatabase(project);
99N/A }
491N/A int idx = dbs.indexOf(db);
491N/A if (idx != -1) {
491N/A db = dbs.get(idx);
58N/A }
77N/A
491N/A if (db.addDirectory(path)) {
491N/A if (idx == -1) {
491N/A dbs.add(db);
77N/A }
491N/A } else {
1327N/A log.warning("Directory does not exist '" + path + "'");
99N/A }
58N/A }
491N/A }
58N/A
491N/A for (final IndexDatabase db : dbs) {
491N/A final boolean optimize = env.isOptimizeDatabase();
491N/A db.addIndexChangedListener(progress);
491N/A executor.submit(new Runnable() {
207N/A
1054N/A @Override
491N/A public void run() {
491N/A try {
491N/A if (update) {
491N/A db.update();
491N/A } else if (optimize) {
491N/A db.optimize();
297N/A }
1264N/A } catch (Throwable e) {
1327N/A log.warning("An error occured while "
1327N/A + (update ? "updating " : "optimizing ")
1327N/A + db + ": " + e.getMessage());
1327N/A log.log(Level.FINE, "run", e);
274N/A }
274N/A }
491N/A });
491N/A }
491N/A }
491N/A
491N/A executor.shutdown();
491N/A while (!executor.isTerminated()) {
491N/A try {
1190N/A // Wait forever
930N/A executor.awaitTermination(999,TimeUnit.DAYS);
583N/A } catch (InterruptedException exp) {
1327N/A log.warning("Received interrupt while waiting for executor to finish: "
1327N/A + exp.getMessage());
1327N/A log.log(Level.FINE, "doIndexerExecution", exp);
491N/A }
583N/A }
1329N/A log.info("Done.");
491N/A }
261N/A
1463N/A /**
1463N/A * Send the configuration of the current thread's runtime environment to
1463N/A * the given host.
1463N/A * @param configHost where to send the configuration. Format: host:port .
1463N/A */
1463N/A public static void sendToConfigHost(String configHost) {
491N/A if (configHost != null) {
491N/A String[] cfg = configHost.split(":");
1327N/A log.log(Level.INFO, "Sending configuration to {0}", configHost);
491N/A if (cfg.length == 2) {
491N/A try {
491N/A InetAddress host = InetAddress.getByName(cfg[0]);
491N/A RuntimeEnvironment.getInstance().writeConfiguration(host, Integer.parseInt(cfg[1]));
491N/A } catch (Exception ex) {
1327N/A log.log(Level.WARNING, "Failed to send configuration to "
1327N/A + configHost
1327N/A + " (is web application server running with opengrok deployed?): "
1327N/A + ex.getMessage());
1327N/A log.log(Level.FINE, "sendToConfigHost", ex);
264N/A }
1327N/A } else if ( cfg.length > 0 ) {
1327N/A StringBuilder sb = new StringBuilder("Syntax errors = ");
491N/A for (String s : cfg) {
1327N/A sb.append('[').append(s).append(']').append(", ");
1190N/A }
1327N/A sb.setLength(sb.length()-2);
1327N/A log.warning(sb.toString());
491N/A }
1329N/A log.info("Done.");
491N/A }
0N/A }
260N/A
260N/A private Indexer() {
260N/A }
0N/A}