1364N/A

1364N/Apackage org.opensolaris.opengrok.index;

1364N/A

1364N/Aimport java.io.File;

1364N/Aimport java.io.IOException;

1364N/Aimport java.net.InetAddress;

1364N/Aimport java.text.ParseException;

1364N/Aimport java.util.ArrayList;

1364N/Aimport java.util.Collections;

1364N/Aimport java.util.Comparator;

1364N/Aimport java.util.List;

1364N/Aimport java.util.concurrent.ExecutorService;

1389N/Aimport java.util.concurrent.Executors;

1364N/Aimport java.util.concurrent.TimeUnit;

1364N/Aimport java.util.logging.Level;

1383N/Aimport java.util.logging.Logger;

1364N/Aimport org.opensolaris.opengrok.Info;

1364N/Aimport org.opensolaris.opengrok.OpenGrokLogger;

1364N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;

1364N/Aimport org.opensolaris.opengrok.configuration.Project;

1364N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;

1364N/Aimport org.opensolaris.opengrok.history.HistoryGuru;

1364N/Aimport org.opensolaris.opengrok.util.Getopt;

1364N/A

1364N/A@SuppressWarnings({"PMD.AvoidPrintStackTrace", "PMD.SystemPrintln"})

1364N/Apublic final class Indexer {

1364N/A

1364N/A private final static String ON = "on";

1383N/A private final static String OFF = "off";

1383N/A private static Indexer index = new Indexer();

1383N/A private static final Logger log = Logger.getLogger(Indexer.class.getName());

1383N/A

1364N/A private static final String DERBY_EMBEDDED_DRIVER =

1364N/A "org.apache.derby.jdbc.EmbeddedDriver";

1364N/A

1364N/A private static final String DERBY_CLIENT_DRIVER =

1364N/A "org.apache.derby.jdbc.ClientDriver";

1364N/A

1364N/A public static Indexer getInstance() {

1364N/A return index;

1364N/A }

1364N/A

1364N/A /**

1364N/A @SuppressWarnings("PMD.UseStringBufferForStringAppends")

1364N/A public static void main(String argv[]) {

1364N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();

1364N/A boolean runIndex = true;

1383N/A boolean update = true;

1364N/A boolean optimizedChanged = false;

1364N/A CommandLineOptions cmdOptions = new CommandLineOptions();

1364N/A

1364N/A if (argv.length == 0) {

1364N/A System.err.println(cmdOptions.getUsage());

1364N/A System.exit(1);

1364N/A } else {

1370N/A boolean searchRepositories = false;

1364N/A ArrayList<String> subFiles = new ArrayList<String>();

1364N/A ArrayList<String> repositories = new ArrayList<String>();

1370N/A String configFilename = null;

1364N/A String configHost = null;

1364N/A boolean addProjects = false;

1370N/A boolean refreshHistory = false;

1364N/A String defaultProject = null;

1364N/A boolean listFiles = false;

1364N/A boolean createDict = false;

1370N/A int noThreads = 2 + (2 * Runtime.getRuntime().availableProcessors());

1364N/A

1364N/A // Parse command line options:

1383N/A Getopt getopt = new Getopt(argv, cmdOptions.getCommandString());

1364N/A

1364N/A try {

1389N/A getopt.parse();

1383N/A } catch (ParseException ex) {

1364N/A System.err.println("OpenGrok: " + ex.getMessage());

1370N/A System.err.println(cmdOptions.getUsage());

1364N/A System.exit(1);

1364N/A }

1364N/A

1364N/A try {

1383N/A int cmd;

1364N/A

1364N/A // We need to read the configuration file first, since we

1383N/A // will try to overwrite options..

1364N/A while ((cmd = getopt.getOpt()) != -1) {

1383N/A if (cmd == 'R') {

1383N/A env.readConfiguration(new File(getopt.getOptarg()));

1370N/A break;

1364N/A }

1364N/A }

1364N/A

1364N/A String databaseDriver = env.getDatabaseDriver();

1370N/A String databaseURL = env.getDatabaseUrl();

1364N/A

1364N/A // Now we can handle all the other options..

1364N/A getopt.reset();

1383N/A while ((cmd = getopt.getOpt()) != -1) {

1383N/A switch (cmd) {

1364N/A case 't':

1383N/A createDict = true;

1383N/A runIndex = false;

1364N/A break;

1370N/A

1364N/A case 'q':

1364N/A env.setVerbose(false);

1370N/A break;

1364N/A case 'e':

1364N/A env.setGenerateHtml(false);

1364N/A break;

1364N/A case 'P':

1383N/A addProjects = true;

1383N/A break;

1383N/A case 'p':

1383N/A defaultProject = getopt.getOptarg();

1389N/A break;

1389N/A case 'c':

1383N/A env.setCtags(getopt.getOptarg());

1383N/A break;

1383N/A case 'w':

1383N/A {

1383N/A String webapp = getopt.getOptarg();

1389N/A if (webapp.charAt(0) != '/' && !webapp.startsWith("http")) {

1389N/A webapp = "/" + webapp;

1389N/A }

1389N/A if (webapp.endsWith("/")) {

1383N/A env.setUrlPrefix(webapp + "s?");

1383N/A } else {

1383N/A env.setUrlPrefix(webapp + "/s?");

1383N/A }

1383N/A }

1389N/A break;

1389N/A case 'W':

1383N/A configFilename = getopt.getOptarg();

1383N/A break;

1383N/A case 'U':

1383N/A configHost = getopt.getOptarg();

1364N/A break;

1364N/A case 'R':

1389N/A // already handled

1364N/A break;

1364N/A case 'n':

1364N/A runIndex = false;

1364N/A break;

1364N/A case 'H':

1364N/A refreshHistory = true;

1364N/A break;

1364N/A case 'h':

1364N/A repositories.add(getopt.getOptarg());

1364N/A break;

1364N/A case 'D':

1364N/A env.setStoreHistoryCacheInDB(true);

1364N/A break;

1364N/A case 'j':

1364N/A databaseDriver = getopt.getOptarg();

1364N/A // Should be a full class name, but we also accept

1364N/A // the shorthands "client" and "embedded". Expand

1364N/A // the shorthands here.

1364N/A if ("client".equals(databaseDriver)) {

1364N/A databaseDriver = DERBY_CLIENT_DRIVER;

1364N/A } else if ("embedded".equals(databaseDriver)) {

1364N/A databaseDriver = DERBY_EMBEDDED_DRIVER;

1364N/A }

1364N/A break;

1364N/A case 'u':

1364N/A databaseURL = getopt.getOptarg();

1364N/A break;

1364N/A case 'r':

1364N/A {

1364N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {

1364N/A env.setRemoteScmSupported(true);

1364N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {

1364N/A env.setRemoteScmSupported(false);

1364N/A } else {

1364N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -r");

1364N/A System.err.println(" Ex: \"-r on\" will allow retrival for remote SCM systems");

1364N/A System.err.println(" \"-r off\" will ignore SCM for remote systems");

1364N/A }

1364N/A }

1364N/A break;

1364N/A case 'O':

1364N/A {

1364N/A boolean oldval = env.isOptimizeDatabase();

1364N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {

1364N/A env.setOptimizeDatabase(true);

1364N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {

1364N/A env.setOptimizeDatabase(false);

1364N/A } else {

1364N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -O");

1364N/A System.err.println(" Ex: \"-O on\" will optimize the database as part of the index generation");

1364N/A System.err.println(" \"-O off\" disable optimization of the index database");

1364N/A }

1364N/A if (oldval != env.isOptimizeDatabase()) {

1364N/A optimizedChanged = true;

1364N/A }

1364N/A }

1364N/A break;

1364N/A case 'v':

1364N/A env.setVerbose(true);

1364N/A break;

1364N/A

1364N/A case 's':

1364N/A {

1364N/A env.setSourceRoot(getopt.getOptarg());

1364N/A File file = env.getSourceRootFile();

1364N/A if (!file.isDirectory()) {

1364N/A System.err.println("ERROR: source root must be a directory: " + file.toString());

1364N/A System.exit(1);

1364N/A }

1364N/A }

1364N/A break;

1364N/A case 'd':

1364N/A {

1364N/A env.setDataRoot(getopt.getOptarg());

1364N/A File file = env.getDataRootFile();

1364N/A if (!file.isDirectory()) {

1364N/A System.err.println("ERROR: data root must be a directory: " + file.toString());

1364N/A System.exit(1);

1364N/A }

1364N/A }

1364N/A break;

1364N/A case 'i':

1364N/A env.getIgnoredNames().add(getopt.getOptarg());

1364N/A break;

1364N/A case 'S':

1364N/A searchRepositories = true;

1364N/A break;

1364N/A case 'Q':

1364N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {

1364N/A env.setQuickContextScan(true);

1364N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {

1364N/A env.setQuickContextScan(false);

1364N/A } else {

1364N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -Q");

1364N/A System.err.println(" Ex: \"-Q on\" will just scan a \"chunk\" of the file and insert \"[..all..]\"");

1364N/A System.err.println(" \"-Q off\" will try to build a more accurate list by reading the complete file.");

1364N/A }

1364N/A

1364N/A break;

1364N/A case 'm': {

1364N/A try {

1364N/A env.setIndexWordLimit(Integer.parseInt(getopt.getOptarg()));

1364N/A } catch (NumberFormatException exp) {

1364N/A System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());

1364N/A System.exit(1);

1364N/A }

1364N/A break;

1364N/A }

1364N/A case 'a':

1364N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {

1364N/A env.setAllowLeadingWildcard(true);

1364N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {

1364N/A env.setAllowLeadingWildcard(false);

1364N/A } else {

1364N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -a");

1364N/A System.err.println(" Ex: \"-a on\" will allow a search to start with a wildcard");

1364N/A System.err.println(" \"-a off\" will disallow a search to start with a wildcard");

1364N/A System.exit(1);

1364N/A }

1364N/A

1364N/A break;

1364N/A

1364N/A case 'A':

1364N/A {

1364N/A String[] arg = getopt.getOptarg().split(":");

1364N/A if (arg.length != 2) {

1364N/A System.err.println("ERROR: You must specify: -A extension:class");

1389N/A System.err.println(" Ex: -A foo:org.opensolaris.opengrok.analysis.c.CAnalyzer");

1364N/A System.err.println(" will use the C analyzer for all files ending with .foo");

1364N/A System.err.println(" Ex: -A c:-");

1364N/A System.err.println(" will disable the c-analyzer for for all files ending with .c");

1364N/A System.exit(1);

949N/A }

949N/A

949N/A arg[0] = arg[0].substring(arg[0].lastIndexOf('.') + 1).toUpperCase();

1294N/A if (arg[1].equals("-")) {

1186N/A AnalyzerGuru.addExtension(arg[0], null);

949N/A break;

949N/A }

1186N/A

949N/A try {

1186N/A AnalyzerGuru.addExtension(

1186N/A arg[0],

1186N/A AnalyzerGuru.findFactory(arg[1]));

1186N/A } catch (Exception e) {

1186N/A System.err.println("Unable to use " + arg[1] +

1186N/A " as a FileAnalyzerFactory");

1186N/A e.printStackTrace();

1186N/A System.exit(1);

1186N/A }

1186N/A }

1186N/A break;

1186N/A case 'L':

1186N/A env.setWebappLAF(getopt.getOptarg());

1186N/A break;

1186N/A case 'T':

1186N/A try {

1186N/A noThreads = Integer.parseInt(getopt.getOptarg());

1186N/A } catch (NumberFormatException exp) {

1186N/A System.err.println("ERROR: Failed to parse argument to \"-T\": " + exp.getMessage());

1186N/A System.exit(1);

1186N/A }

1186N/A break;

949N/A case 'z':

949N/A try {

1186N/A env.setScanningDepth(Integer.parseInt(getopt.getOptarg()));

1186N/A } catch (NumberFormatException exp) {

1186N/A System.err.println("ERROR: Failed to parse argument to \"-z\": " + exp.getMessage());

1186N/A System.exit(1);

1186N/A }

1186N/A break;

1186N/A case 'l':

1186N/A if (getopt.getOptarg().equalsIgnoreCase(ON)) {

1390N/A env.setUsingLuceneLocking(true);

1390N/A } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {

1390N/A env.setUsingLuceneLocking(false);

1390N/A } else {

1390N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -l");

1186N/A System.err.println(" Ex: \"-l on\" will enable locks in Lucene");

1186N/A System.err.println(" \"-l off\" will disable locks in Lucene");

1390N/A }

1390N/A break;

1390N/A case 'V':

1390N/A System.out.println(Info.getFullVersion());

1186N/A System.exit(0);

1186N/A break;

1186N/A

1186N/A case '?':

1186N/A System.err.println(cmdOptions.getUsage());

1186N/A System.exit(0);

1186N/A break;

1186N/A

1186N/A default:

1186N/A System.err.println("Internal Error - Unimplemented cmdline option: " + (char) cmd);

1186N/A System.exit(1);

1186N/A }

1186N/A }

1388N/A

1186N/A int optind = getopt.getOptind();

1186N/A if (optind != -1) {

949N/A while (optind < argv.length) {

949N/A subFiles.add(argv[optind]);

1186N/A ++optind;

1186N/A }

1186N/A }

1186N/A

1186N/A if (env.storeHistoryCacheInDB()) {

1389N/A // The default database driver is Derby's client driver.

1389N/A if (databaseDriver == null) {

1389N/A databaseDriver = DERBY_CLIENT_DRIVER;

1186N/A }

1389N/A

1186N/A // The default URL depends on the database driver.

1389N/A if (databaseURL == null) {

1186N/A StringBuilder defaultURL = new StringBuilder();

1186N/A defaultURL.append("jdbc:derby:");

1186N/A if (databaseDriver.equals(DERBY_EMBEDDED_DRIVER)) {

1186N/A defaultURL

1186N/A .append(env.getDataRootPath())

1186N/A .append(File.separator);

1186N/A } else {

1186N/A defaultURL.append("//localhost/");

1186N/A }

1186N/A defaultURL.append("cachedb;create=true");

1186N/A databaseURL = defaultURL.toString();

1390N/A }

1390N/A }

1186N/A

1186N/A env.setDatabaseDriver(databaseDriver);

1186N/A env.setDatabaseUrl(databaseURL);

1186N/A

1186N/A getInstance().prepareIndexer(env, searchRepositories, addProjects,

1186N/A defaultProject, configFilename, refreshHistory,

1186N/A listFiles, createDict, subFiles, repositories);

949N/A if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {

949N/A IndexChangedListener progress = new DefaultIndexChangedListener();

1186N/A getInstance().doIndexerExecution(update, noThreads, subFiles,

1390N/A progress);

1390N/A }

1390N/A getInstance().sendToConfigHost(env, configHost);

1390N/A } catch (IndexerException ex) {

1390N/A OpenGrokLogger.getLogger().log(Level.SEVERE, "Exception running indexer", ex);

1390N/A System.err.println(cmdOptions.getUsage());

1390N/A System.exit(1);

1390N/A } catch (IOException ioe) {

1390N/A System.err.println("Got IOException " + ioe);

1186N/A OpenGrokLogger.getLogger().log(Level.SEVERE, "Exception running indexer", ioe);

1186N/A System.exit(1);

1186N/A }

1390N/A }

1390N/A

1186N/A }

1186N/A

1186N/A public void prepareIndexer(RuntimeEnvironment env,

1390N/A boolean searchRepositories,

1390N/A boolean addProjects,

1390N/A String defaultProject,

1390N/A String configFilename,

1390N/A boolean refreshHistory,

1390N/A boolean listFiles,

1390N/A boolean createDict,

1390N/A List<String> subFiles,

1390N/A List<String> repositories) throws IndexerException, IOException {

1390N/A

1390N/A if (env.getDataRootPath() == null) {

1390N/A throw new IndexerException("ERROR: Please specify a DATA ROOT path");

1390N/A }

1390N/A

1186N/A if (env.getSourceRootFile() == null) {

1186N/A throw new IndexerException("ERROR: please specify a SRC_ROOT with option -s !");

1186N/A }

1186N/A

1186N/A if (!env.validateExuberantCtags()) {

1186N/A throw new IndexerException("Didn't find Exuberant Ctags");

1186N/A }

1186N/A

1186N/A if (searchRepositories) {

1186N/A if (env.isVerbose()) {

1186N/A System.out.println("Scanning for repositories...");

1186N/A }

1186N/A long start = System.currentTimeMillis();

1186N/A HistoryGuru.getInstance().addRepositories(env.getSourceRootPath());

1186N/A long time = (System.currentTimeMillis() - start) / 1000;

1186N/A if (env.isVerbose()) {

1186N/A System.out.println("Done searching for repositories (" + time + "s)");

1186N/A }

1186N/A }

1186N/A

1186N/A if (addProjects) {

1186N/A File files[] = env.getSourceRootFile().listFiles();

1186N/A List<Project> projects = env.getProjects();

1186N/A projects.clear();

1186N/A for (File file : files) {

1186N/A if (!file.getName().startsWith(".") && file.isDirectory()) {

1186N/A Project p = new Project();

1186N/A String name = file.getName();

1186N/A p.setDescription(name);

1186N/A p.setPath("/" + name);

1186N/A projects.add(p);

1186N/A }

1186N/A }

1186N/A

1186N/A // The projects should be sorted...

1186N/A Collections.sort(projects, new Comparator<Project>() {

1469N/A

1469N/A public int compare(Project p1, Project p2) {

1469N/A String s1 = p1.getDescription();

1389N/A String s2 = p2.getDescription();

1186N/A

1186N/A int ret;

1186N/A if (s1 == null) {

1186N/A ret = (s2 == null) ? 0 : 1;

1389N/A } else {

949N/A ret = s1.compareTo(s2);

1186N/A }

949N/A return ret;

1186N/A }

1186N/A });

1186N/A }

1186N/A

1186N/A if (defaultProject != null) {

1186N/A for (Project p : env.getProjects()) {

1186N/A if (p.getPath().equals(defaultProject)) {

1186N/A env.setDefaultProject(p);

949N/A break;

1389N/A }

1389N/A }

1389N/A }

1389N/A

1389N/A if (configFilename != null) {

1186N/A if (env.isVerbose()) {

1389N/A System.out.println("Writing configuration to " + configFilename);

1389N/A System.out.flush();

1186N/A }

1186N/A env.writeConfiguration(new File(configFilename));

1389N/A if (env.isVerbose()) {

1186N/A System.out.println("Done...");

1389N/A System.out.flush();

1186N/A }

1186N/A }

1186N/A

1186N/A if (refreshHistory) {

1186N/A HistoryGuru.getInstance().createCache();

1186N/A } else if (repositories != null && !repositories.isEmpty()) {

1186N/A HistoryGuru.getInstance().createCache(repositories);

949N/A }

1186N/A

1186N/A if (listFiles) {

1186N/A IndexDatabase.listAllFiles(subFiles);

1186N/A }

1186N/A

1186N/A if (createDict) {

1186N/A IndexDatabase.listFrequentTokens(subFiles);

1186N/A }

1186N/A }

1186N/A

1186N/A public void doIndexerExecution(final boolean update, int noThreads, List<String> subFiles,

1186N/A IndexChangedListener progress)

1390N/A throws IOException {

1186N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();

1186N/A env.register();

1186N/A log.info("Starting indexExecution");

1186N/A

1186N/A ExecutorService executor = Executors.newFixedThreadPool(noThreads);

1186N/A

1186N/A if (subFiles == null || subFiles.isEmpty()) {

1186N/A if (update) {

1186N/A IndexDatabase.updateAll(executor, progress);

1186N/A } else if (env.isOptimizeDatabase()) {

1186N/A IndexDatabase.optimizeAll(executor);

1186N/A }

1186N/A } else {

1186N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();

1390N/A

1186N/A for (String path : subFiles) {

1186N/A Project project = Project.getProject(path);

1186N/A if (project == null && env.hasProjects()) {

1186N/A System.err.println("Warning: Could not find a project for \"" + path + "\"");

1186N/A } else {

1186N/A IndexDatabase db;

1390N/A if (project == null) {

1390N/A db = new IndexDatabase();

1186N/A } else {

1186N/A db = new IndexDatabase(project);

1186N/A }

1390N/A int idx = dbs.indexOf(db);

1390N/A if (idx != -1) {

1186N/A db = dbs.get(idx);

1186N/A }

949N/A

1476N/A if (db.addDirectory(path)) {

1476N/A if (idx == -1) {

1469N/A dbs.add(db);

1423N/A }

1423N/A } else {

1423N/A System.err.println("Warning: Directory does not exist \"" + path + "\"");

1186N/A }

1423N/A }

1423N/A }

1423N/A

1423N/A for (final IndexDatabase db : dbs) {

1423N/A final boolean optimize = env.isOptimizeDatabase();

1186N/A db.addIndexChangedListener(progress);

1294N/A executor.submit(new Runnable() {

1186N/A

1186N/A public void run() {

1423N/A try {

1423N/A if (update) {

1423N/A db.update();

1423N/A } else if (optimize) {

949N/A db.optimize();

1423N/A }

1423N/A } catch (Exception e) {

1423N/A if (update) {

1423N/A OpenGrokLogger.getLogger().log(Level.WARNING, "An error occured while updating index", e);

1423N/A } else {

1423N/A OpenGrokLogger.getLogger().log(Level.WARNING, "An error occured while optimizing index", e);

1423N/A }

1186N/A e.printStackTrace();

1186N/A }

1186N/A }

1186N/A });

1186N/A }

1186N/A }

1186N/A

1186N/A executor.shutdown();

1186N/A while (!executor.isTerminated()) {

1388N/A try {

1388N/A // Wait forever

1186N/A executor.awaitTermination(999,TimeUnit.DAYS);

1186N/A } catch (InterruptedException exp) {

1186N/A OpenGrokLogger.getLogger().log(Level.WARNING, "Received interrupt while waiting for executor to finish", exp);

1186N/A }

1186N/A }

1388N/A }

1388N/A

1186N/A public void sendToConfigHost(RuntimeEnvironment env, String configHost) {

1186N/A if (configHost != null) {

1388N/A String[] cfg = configHost.split(":");

1388N/A if (env.isVerbose()) {

1186N/A log.info("Send configuration to: " + configHost);

1186N/A }

1388N/A

1186N/A if (cfg.length == 2) {

1186N/A try {

1388N/A InetAddress host = InetAddress.getByName(cfg[0]);

1186N/A RuntimeEnvironment.getInstance().writeConfiguration(host, Integer.parseInt(cfg[1]));

1186N/A } catch (Exception ex) {

1186N/A log.log(Level.SEVERE, "Failed to send configuration to " + configHost+" (is web application server running with opengrok deployed?)", ex);

1388N/A }

1388N/A } else {

1186N/A System.err.println("Syntax error: ");

1388N/A for (String s : cfg) {

1186N/A System.err.print("[" + s + "]");

1186N/A }

1186N/A System.err.println();

1186N/A }

1186N/A if (env.isVerbose()) {

1186N/A log.info("Configuration update routine done, check previous output for errors.");

1186N/A }

1388N/A }

1388N/A }

1186N/A

1388N/A private Indexer() {

1186N/A }

1186N/A}

1388N/A