Indexer.java revision 1e4f2633aeeca23eeb54f4007ab4f54565b499a7
cd348e325366620fe047edcc849e3c9424828599Peter Bray * CDDL HEADER START
cd348e325366620fe047edcc849e3c9424828599Peter Bray * The contents of this file are subject to the terms of the
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * Common Development and Distribution License (the "License").
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal * You may not use this file except in compliance with the License.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * See LICENSE.txt included in this distribution for the specific
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * language governing permissions and limitations under the License.
983523cf73bc85cce6282cb5aa78b60f6bcd959fLubos Kosco * When distributing Covered Code, include this CDDL HEADER in each
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * file and include the License file at LICENSE.txt.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * If applicable, add the following below this CDDL HEADER, with the
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * fields enclosed by brackets "[]" replaced with your own identifying
c577d2f6c082eaff9af5bc997d12f3d3bcef537cPeter Bray * information: Portions Copyright [yyyy] [name of copyright owner]
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * CDDL HEADER END
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * Portions Copyright 2011 Jens Elkner.
65df1e87266301e243dcead8082ca6255abea666Vladimir Kotalimport org.opensolaris.opengrok.analysis.AnalyzerGuru;
65df1e87266301e243dcead8082ca6255abea666Vladimir Kotalimport org.opensolaris.opengrok.configuration.Configuration;
425278cfacbc73f1e955ab6016f206fc5ed93ccbVladimir Kotalimport org.opensolaris.opengrok.configuration.Project;
425278cfacbc73f1e955ab6016f206fc5ed93ccbVladimir Kotalimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
65df1e87266301e243dcead8082ca6255abea666Vladimir Kotalimport org.opensolaris.opengrok.history.HistoryException;
65df1e87266301e243dcead8082ca6255abea666Vladimir Kotalimport org.opensolaris.opengrok.history.HistoryGuru;
65df1e87266301e243dcead8082ca6255abea666Vladimir Kotalimport org.opensolaris.opengrok.history.Repository;
425278cfacbc73f1e955ab6016f206fc5ed93ccbVladimir Kotalimport org.opensolaris.opengrok.history.RepositoryFactory;
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotalimport org.opensolaris.opengrok.history.RepositoryInfo;
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotalimport org.opensolaris.opengrok.logger.LoggerFactory;
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotalimport org.opensolaris.opengrok.logger.LoggerUtil;
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotalimport org.opensolaris.opengrok.util.Executor;
6ce0623fa4ef95af9d77700a1c9c19ec1a919326Guillaume Smetimport org.opensolaris.opengrok.util.Statistics;
56c25decc0427c204cd35856e521ddf28337e75dLubos Kosco * Creates and updates an inverted source index as well as generates Xref, file
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal * stats etc., if specified in the options
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal@SuppressWarnings({"PMD.AvoidPrintStackTrace", "PMD.SystemPrintln"})
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotalpublic final class Indexer {
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal private static final Logger LOGGER = LoggerFactory.getLogger(Indexer.class);
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal /* tunables for -r (history for remote repositories) */
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal private static final String DIRBASED = "dirbased";
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal private static final String UIONLY = "uionly";
983523cf73bc85cce6282cb5aa78b60f6bcd959fLubos Kosco private static final Indexer index = new Indexer();
983523cf73bc85cce6282cb5aa78b60f6bcd959fLubos Kosco private static final String DERBY_EMBEDDED_DRIVER
983523cf73bc85cce6282cb5aa78b60f6bcd959fLubos Kosco = "org.apache.derby.jdbc.EmbeddedDriver";
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal = "org.apache.derby.jdbc.ClientDriver";
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal private static void A_usage() {
425278cfacbc73f1e955ab6016f206fc5ed93ccbVladimir Kotal System.err.println("ERROR: You must specify: -A .extension:class or -A prefix.:class");
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal System.err.println(" Ex: -A .foo:org.opensolaris.opengrok.analysis.c.CAnalyzer");
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println(" will use the C analyzer for all files ending with .foo");
3b0448fdd830b8d04c6a71511e5d26a4fc3b5b80Lubos Kosco System.err.println(" Ex: -A bar.:org.opensolaris.opengrok.analysis.c.CAnalyzer");
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println(" will use the C analyzer for all files starting with bar.");
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println(" will disable the c-analyzer for for all files ending with .c");
3a4816d2417e1abe89a913616de36f200793bea3Vladimir Kotal * Program entry point
610915d59310c9aba810740e858ba64c5e3e6a2fVladimir Kotal * @param argv argument vector
fbf97ea1786d1e25add88bbfb91810170473bc9fLubos Kosco @SuppressWarnings("PMD.UseStringBufferForStringAppends")
fbf97ea1786d1e25add88bbfb91810170473bc9fLubos Kosco Statistics stats = new Statistics(); //this won't count JVM creation though
fbf97ea1786d1e25add88bbfb91810170473bc9fLubos Kosco boolean runIndex = true;
fbf97ea1786d1e25add88bbfb91810170473bc9fLubos Kosco boolean update = true;
f21b682cd9b414738a4f5a38b56f6682e537e1d2Trond Norbye boolean optimizedChanged = false;
3b0448fdd830b8d04c6a71511e5d26a4fc3b5b80Lubos Kosco CommandLineOptions cmdOptions = new CommandLineOptions();
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal boolean searchRepositories = false;
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal ArrayList<String> subFiles = new ArrayList<>();
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal ArrayList<String> subFilesList = new ArrayList<>();
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal ArrayList<String> repositories = new ArrayList<>();
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal HashSet<String> allowedSymlinks = new HashSet<>();
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal boolean addProjects = false;
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal boolean refreshHistory = false;
0ca9a2c194523c517c3aafe5758e217ac88d6baaLubos Kosco boolean listFiles = false;
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray boolean listRepos = false;
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray boolean createDict = false;
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray int noThreads = 2 + (2 * Runtime.getRuntime().availableProcessors());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Parse command line options:
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray Getopt getopt = new Getopt(argv, cmdOptions.getCommandString());
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println("OpenGrok: " + ex.getMessage());
cd348e325366620fe047edcc849e3c9424828599Peter Bray // We need to read the configuration file first, since we
cd348e325366620fe047edcc849e3c9424828599Peter Bray // will try to overwrite options..
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray cfg = Configuration.read(new File(getopt.getOptarg()));
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray // Now we can handle all the other options..
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray boolean prefix = false;
cd348e325366620fe047edcc849e3c9424828599Peter Bray arg[0] = arg[0].substring(0, arg[0].lastIndexOf('.')).toUpperCase();
cd348e325366620fe047edcc849e3c9424828599Peter Bray arg[0] = arg[0].substring(arg[0].lastIndexOf('.') + 1).toUpperCase();
c842732324ee4c74ede17887ad1f0dcdc4364a2cLubos Kosco } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
c842732324ee4c74ede17887ad1f0dcdc4364a2cLubos Kosco LOGGER.log(Level.SEVERE, "Unable to use {0} as a FileAnalyzerFactory", arg[1]);
c842732324ee4c74ede17887ad1f0dcdc4364a2cLubos Kosco LOGGER.log(Level.SEVERE, "Stack: ", e.fillInStackTrace());
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal LOGGER.log(Level.SEVERE, "Unable to use {0} as a FileAnalyzerFactory", arg[1]);
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal LOGGER.log(Level.SEVERE, "Stack: ", e.fillInStackTrace());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -a");
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println(" Ex: \"-a on\" will allow a search to start with a wildcard");
d961aa46ea0d50fed47802497e45226b1965b12dVladimir Kotal System.err.println(" \"-a off\" will disallow a search to start with a wildcard");
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray System.err.println("ERROR: Cannot create data root");
5762c9f28c2246777be0e9d49cb29d9c0f49146dLubos Kosco System.err.println("ERROR: Data root must be a directory");
2cf31ec93bd5d8a2efeab511ce051da51e69aedaLubos Kosco // Should be a full class name, but we also accept
2cf31ec93bd5d8a2efeab511ce051da51e69aedaLubos Kosco // the shorthands "client" and "embedded". Expand
2cf31ec93bd5d8a2efeab511ce051da51e69aedaLubos Kosco // the shorthands here.
2cf31ec93bd5d8a2efeab511ce051da51e69aedaLubos Kosco case "client":
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray case "embedded":
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal if (getopt.getOptarg().equalsIgnoreCase(ON)) {
6c8465e3b4611cb632cba9b0572e3e3737c8c341Vladimir Kotal } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -l");
6ce0623fa4ef95af9d77700a1c9c19ec1a919326Guillaume Smet System.err.println(" Ex: \"-l on\" will enable locks in Lucene");
6ce0623fa4ef95af9d77700a1c9c19ec1a919326Guillaume Smet System.err.println(" \"-l off\" will disable locks in Lucene");
6ce0623fa4ef95af9d77700a1c9c19ec1a919326Guillaume Smet cfg.setRamBufferSize(Double.parseDouble(getopt.getOptarg()));
cd71fb134e037849c77364b50940b1870c4684ceVladimir Kotal System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -O");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println(" Ex: \"-O on\" will optimize the database as part of the index generation");
2b024356b830395446c55f50f9f724a63612e578Lubos Kosco System.err.println(" \"-O off\" disable optimization of the index database");
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray String CTagsExtraOptionsFile = getopt.getOptarg();
4e854c69a0484765dcd27b0e837898c8b6969beaTrond Norbye File CTagsFile = new File(CTagsExtraOptionsFile);
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray if (!(CTagsFile.isFile() && CTagsFile.canRead())) {
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray + "' not found for the -o option");
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal + "options for ctags: " + CTagsExtraOptionsFile);
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal cfg.setCTagsExtraOptionsFile(CTagsExtraOptionsFile);
3ba66fbb56ef22f183da783a1b2718280c357a4eStanislav Kozina if (getopt.getOptarg().equalsIgnoreCase(ON)) {
3ba66fbb56ef22f183da783a1b2718280c357a4eStanislav Kozina } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -Q");
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println(" Ex: \"-Q on\" will just scan a \"chunk\" of the file and insert \"[..all..]\"");
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println(" \"-Q off\" will try to build a more accurate list by reading the complete file.");
cd348e325366620fe047edcc849e3c9424828599Peter Bray LoggerUtil.setBaseConsoleLogLevel(Level.WARNING);
c577d2f6c082eaff9af5bc997d12f3d3bcef537cPeter Bray // already handled
c577d2f6c082eaff9af5bc997d12f3d3bcef537cPeter Bray cfg.setRemoteScmSupported(Configuration.RemoteSCM.ON);
8f8c3f4555e5aa3160f03f2e9c55ddbd3381357bLubos Kosco } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
d0767114e1a949e4a42358f5aeaa08590b87cd80Trond Norbye cfg.setRemoteScmSupported(Configuration.RemoteSCM.OFF);
3a5046f0538ba9fb3a9429199544a9f4b93d9a4dLubos Kosco } else if (getopt.getOptarg().equalsIgnoreCase(DIRBASED)) {
3a5046f0538ba9fb3a9429199544a9f4b93d9a4dLubos Kosco cfg.setRemoteScmSupported(Configuration.RemoteSCM.DIRBASED);
3a5046f0538ba9fb3a9429199544a9f4b93d9a4dLubos Kosco } else if (getopt.getOptarg().equalsIgnoreCase(UIONLY)) {
3a5046f0538ba9fb3a9429199544a9f4b93d9a4dLubos Kosco cfg.setRemoteScmSupported(Configuration.RemoteSCM.UIONLY);
cd348e325366620fe047edcc849e3c9424828599Peter Bray System.err.println("ERROR: You should pass either \"on\" or \"off\" or \"uionly\" as argument to -r");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println(" Ex: \"-r on\" will allow retrieval for remote SCM systems");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println(" \"-r off\" will ignore SCM for remote systems");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println(" \"-r dirbased\" will allow retrieval during history index "
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray + "only for repositories which allow getting history for directories");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println(" \"-r uionly\" will support remote SCM for UI only");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray cfg.setSourceRoot(sourceRoot.getCanonicalPath());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray noThreads = Integer.parseInt(getopt.getOptarg());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println("ERROR: Failed to parse argument to \"-T\": "
c83dfde6b364917fa8ed28142d509a7c29a4da68Vladimir Kotal System.err.println("ERROR: Failed to parse argument to \"-t\": "
d0767114e1a949e4a42358f5aeaa08590b87cd80Trond Norbye if (webapp.charAt(0) != '/' && !webapp.startsWith("http")) {
f9e7e93ed37cbcb0a111b6f997bb75d567c31c0bVladimir Kotal cfg.setScanningDepth(Integer.parseInt(getopt.getOptarg()));
f9e7e93ed37cbcb0a111b6f997bb75d567c31c0bVladimir Kotal System.err.println("ERROR: Failed to parse argument to \"-z\": "
f9e7e93ed37cbcb0a111b6f997bb75d567c31c0bVladimir Kotal System.err.println("Internal Error - Unimplemented cmdline option: " + (char) cmd);
f9e7e93ed37cbcb0a111b6f997bb75d567c31c0bVladimir Kotal List<Class<? extends Repository>> repositoryClasses
f9e7e93ed37cbcb0a111b6f997bb75d567c31c0bVladimir Kotal for (Class<? extends Repository> clazz : repositoryClasses) {
f9e7e93ed37cbcb0a111b6f997bb75d567c31c0bVladimir Kotal Field f = clazz.getDeclaredField("CMD_PROPERTY_KEY");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // don't care
4e854c69a0484765dcd27b0e837898c8b6969beaTrond Norbye //logging starts here
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal String fn = LoggerUtil.getFileHandlerPattern();
c276b1ec9722ee95a86a4a381b39c5f405fc1cc4Vladimir Kotal System.out.println("Logging filehandler pattern: " + fn);
4e854c69a0484765dcd27b0e837898c8b6969beaTrond Norbye // The default database driver is Derby's client driver.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // The default URL depends on the database driver.
a3a5f5c0500064f4c6ac3658b1e3e4d838912329Lubos Kosco if (databaseDriver.equals(DERBY_EMBEDDED_DRIVER)) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // automatically allow symlinks that are directly in source root
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (!projectDir.getCanonicalPath().equals(projectDir.getAbsolutePath())) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray allowedSymlinks.add(projectDir.getAbsolutePath());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray allowedSymlinks.addAll(cfg.getAllowedSymlinks());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Assemble the unprocessed command line arguments (possibly
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // a list of paths). This will be used to perform more fine
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // grained checking in invalidateRepositories().
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray subFilesList.add(cfg.getSourceRoot() + argv[optind++]);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Set updated configuration in RuntimeEnvironment.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray RuntimeEnvironment env = RuntimeEnvironment.getInstance();
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * Add paths to directories under source root. If projects
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * are enabled the path should correspond to a project because
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * project path is necessary to correctly set index directory
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * (otherwise the index files will end up in index data root
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * directory and not per project data root directory).
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * For the check we need to have 'env' already set.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (srcPath==null) { System.err.println("Error getting source root from environment. Exiting.");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // The paths need to correspond to a project.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray + " does not correspond to a project");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (!subFilesList.isEmpty() && subFiles.isEmpty()) {
83dcb435c5eeeb8d0d0d906f5d25a1cb1db5c40bLubos Kosco System.err.println("None of the paths were added, exiting");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Issue a warning when JDBC is used with renamed file handling.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // This causes heavy slowdown when used with JavaDB (issue #774).
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (env.isHandleHistoryOfRenamedFiles() && cfg.isHistoryCacheInDB()) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.out.println("History stored in DB and renamed file "
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray + "handling is on - possible performance degradation");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Get history first.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray getInstance().prepareIndexer(env, searchRepositories, addProjects,
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // And now index it all.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray IndexChangedListener progress = new DefaultIndexChangedListener();
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray getInstance().doIndexerExecution(update, noThreads, subFiles,
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Finally send new configuration to the web application.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.log(Level.SEVERE, "Exception running indexer", ex);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray } catch (Throwable e) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.err.println("Exception: " + e.getLocalizedMessage());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.log(Level.SEVERE, "Unexpected Exception", e);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * This is the first phase of the indexing where history cache is being
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * generated for repositories (at least for those which support getting
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray * history per directory).
56d93b0c761868f813ac0bc0b5bc21a7a9fefd89Vladimir Kotal * PMD wants us to use length() > 0 && charAt(0) instead of startsWith()
56d93b0c761868f813ac0bc0b5bc21a7a9fefd89Vladimir Kotal * for performance. We prefer clarity over performance here, so silence it.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray public void prepareIndexer(RuntimeEnvironment env,
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray boolean listRepoPaths) throws IndexerException, IOException {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray throw new IndexerException("ERROR: Please specify a DATA ROOT path");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray throw new IndexerException("ERROR: please specify a SRC_ROOT with option -s !");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (zapCache.isEmpty() && !env.validateExuberantCtags()) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray throw new IndexerException("Didn't find Exuberant Ctags");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray throw new IndexerException("Internal error, zapCache shouldn't be null");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (searchRepositories || listRepoPaths || !zapCache.isEmpty()) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.log(Level.INFO, "Scanning for repositories...");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray if (refreshHistory == true) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray HistoryGuru.getInstance().addRepositories(env.getSourceRootPath());
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray long time = (System.currentTimeMillis() - start) / 1000;
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.log(Level.INFO, "Done scanning for repositories ({0}s)", time);
0b2998be561e7bf5e3479d686a5af36f712b0d9aVladimir Kotal List<RepositoryInfo> repos = env.getRepositories();
0b2998be561e7bf5e3479d686a5af36f712b0d9aVladimir Kotal System.out.println("No repositories found.");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.out.println("Repositories in " + prefix + ":");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray for (RepositoryInfo info : env.getRepositories()) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray System.out.println(dir.substring(prefix.length()));
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray HashSet<String> toZap = new HashSet<>(zapCache.size() << 1);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray boolean all = false;
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray for (RepositoryInfo info : env.getRepositories()) {
c577d2f6c082eaff9af5bc997d12f3d3bcef537cPeter Bray LOGGER.log(Level.WARNING, "Clearing history cache failed: {0}", e.getLocalizedMessage());
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal File files[] = env.getSourceRootFile().listFiles();
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal // Keep a copy of the old project list so that we can preserve
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal // the customization of existing projects.
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal Map<String, Project> oldProjects = new HashMap<>();
83439b4ed8fe40097dc3f2c05168d26bd7926159Vladimir Kotal // Add a project for each top-level directory in source root.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // This is an existing object. Reuse the old project,
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // possibly with customizations, instead of creating a
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // new with default values.
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray } else if (!name.startsWith(".") && file.isDirectory()) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Found a new directory with no matching project, so
477c09a2656e6a2c1075425ad81e61d594164fa9Lubos Kosco // create a new project with default properties.
3aa0947feb67d3e8292d84776638be98dd97fdc3Lubos Kosco p.setTabSize(env.getConfiguration().getTabSize());
3aa0947feb67d3e8292d84776638be98dd97fdc3Lubos Kosco // The projects should be sorted...
3aa0947feb67d3e8292d84776638be98dd97fdc3Lubos Kosco Collections.sort(projects, new Comparator<Project>() {
0b2998be561e7bf5e3479d686a5af36f712b0d9aVladimir Kotal LOGGER.log(Level.INFO, "Writing configuration to {0}", configFilename);
0b2998be561e7bf5e3479d686a5af36f712b0d9aVladimir Kotal env.writeConfiguration(new File(configFilename));
5762c9f28c2246777be0e9d49cb29d9c0f49146dLubos Kosco if (repositories != null && !repositories.isEmpty()) {
477c09a2656e6a2c1075425ad81e61d594164fa9Lubos Kosco LOGGER.log(Level.INFO, "Generating history cache for specified repositories ...");
477c09a2656e6a2c1075425ad81e61d594164fa9Lubos Kosco HistoryGuru.getInstance().createCache(repositories);
56d93b0c761868f813ac0bc0b5bc21a7a9fefd89Vladimir Kotal LOGGER.log(Level.INFO, "Generating history cache for all repositories ...");
caf0d2e1c473e03780acaa86e2a73ff953802a8bVladimir Kotal * This is the second phase of the indexer which generates Lucene index
caf0d2e1c473e03780acaa86e2a73ff953802a8bVladimir Kotal * by passing source code files through Exuberant ctags, generating xrefs
caf0d2e1c473e03780acaa86e2a73ff953802a8bVladimir Kotal * and storing data from the source files in the index (along with history,
caf0d2e1c473e03780acaa86e2a73ff953802a8bVladimir Kotal public void doIndexerExecution(final boolean update, int noThreads, List<String> subFiles,
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal RuntimeEnvironment env = RuntimeEnvironment.getInstance().register();
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal ExecutorService executor = Executors.newFixedThreadPool(noThreads);
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal if (subFiles == null || subFiles.isEmpty()) {
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal LOGGER.log(Level.WARNING, "Directory does not exist \"{0}\"", path);
bc5565fc58603964988b42b6aee40e246f35d94fVladimir Kotal final boolean optimize = env.isOptimizeDatabase();
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray public void run() {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray } else if (optimize) {
cd348e325366620fe047edcc849e3c9424828599Peter Bray } catch (Throwable e) {
cd348e325366620fe047edcc849e3c9424828599Peter Bray LOGGER.log(Level.SEVERE, "An error occured while "
d280c5e286f5b98be13237f52281ae5afdcf51b9Peter Bray + " index", e);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // Wait forever
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.log(Level.WARNING, "Received interrupt while waiting for executor to finish", exp);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // It can happen that history index is not done in prepareIndexer()
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray // but via db.update() above in which case we must make sure the
56d93b0c761868f813ac0bc0b5bc21a7a9fefd89Vladimir Kotal // thread pool for renamed file handling is destroyed.
56d93b0c761868f813ac0bc0b5bc21a7a9fefd89Vladimir Kotal RuntimeEnvironment.destroyRenamedHistoryExecutor();
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray elapsed.report(LOGGER, "Done indexing data of all repositories");
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray public void sendToConfigHost(RuntimeEnvironment env, String configHost) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.log(Level.INFO, "Send configuration to: {0}", configHost);
561ec0545f30e44833b6def5168abbd3ab5cf65aVladimir Kotal InetAddress host = InetAddress.getByName(cfg[0]);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray env.writeConfiguration(host, Integer.parseInt(cfg[1]));
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray } catch (NumberFormatException | IOException ex) {
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.log(Level.SEVERE, "Failed to send configuration to "
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray + configHost + " (is web application server running with opengrok deployed?)", ex);
d2a02e104622a26dd90fa88f4f17188f2039809fPeter Bray LOGGER.info("Configuration update routine done, check log output for errors.");