0N/A

0N/Apackage org.opensolaris.opengrok.index;

65N/A

125N/Aimport java.io.BufferedReader;

125N/Aimport java.io.File;

125N/Aimport java.io.FileReader;

125N/Aimport java.io.IOException;

58N/Aimport java.net.InetAddress;

77N/Aimport java.text.ParseException;

125N/Aimport java.util.ArrayList;

125N/Aimport java.util.Collections;

125N/Aimport java.util.Comparator;

125N/Aimport java.util.List;

261N/Aimport java.util.concurrent.ExecutorService;

261N/Aimport java.util.concurrent.Executors;

312N/Aimport java.util.logging.Level;

312N/Aimport java.util.logging.Logger;

428N/Aimport org.opensolaris.opengrok.OpenGrokLogger;

126N/Aimport org.opensolaris.opengrok.analysis.AnalyzerGuru;

58N/Aimport org.opensolaris.opengrok.configuration.Project;

394N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;

8N/Aimport org.opensolaris.opengrok.history.HistoryGuru;

77N/Aimport org.opensolaris.opengrok.util.Getopt;

0N/A

0N/Apublic class Indexer {

312N/A

312N/A private static Indexer index = new Indexer();

320N/A private static final Logger log = Logger.getLogger(Indexer.class.getName());

312N/A

312N/A public static Indexer getInstance() {

312N/A return index;

312N/A }

65N/A /**

0N/A public static void main(String argv[]) {

30N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();

58N/A boolean runIndex = true;

312N/A boolean update = true;

312N/A boolean optimizedChanged = false;

260N/A CommandLineOptions cmdOptions = new CommandLineOptions();

112N/A

428N/A if (argv.length == 0) {

376N/A System.err.println(cmdOptions.getUsage());

376N/A System.exit(1);

0N/A } else {

11N/A boolean searchRepositories = false;

0N/A ArrayList<String> subFiles = new ArrayList<String>();

240N/A ArrayList<String> repositories = new ArrayList<String>();

58N/A String configFilename = null;

58N/A String configHost = null;

58N/A boolean addProjects = false;

58N/A boolean refreshHistory = false;

77N/A String defaultProject = null;

207N/A boolean listFiles = false;

207N/A boolean createDict = false;

261N/A int noThreads = Runtime.getRuntime().availableProcessors();

260N/A

77N/A // Parse command line options:

260N/A Getopt getopt = new Getopt(argv, cmdOptions.getCommandString());

112N/A

77N/A try {

77N/A getopt.parse();

77N/A } catch (ParseException ex) {

77N/A System.err.println("OpenGrok: " + ex.getMessage());

260N/A System.err.println(cmdOptions.getUsage());

77N/A System.exit(1);

77N/A }

77N/A

0N/A try{

77N/A int cmd;

111N/A

111N/A // We need to read the configuration file first, since we

111N/A // will try to overwrite options..

111N/A while ((cmd = getopt.getOpt()) != -1) {

111N/A if (cmd == 'R') {

111N/A env.readConfiguration(new File(getopt.getOptarg()));

111N/A break;

111N/A }

111N/A }

111N/A

111N/A // Now we can handle all the other options..

111N/A getopt.reset();

77N/A while ((cmd = getopt.getOpt()) != -1) {

77N/A switch (cmd) {

77N/A case 't':

207N/A createDict = true;

207N/A runIndex = false;

77N/A break;

77N/A

99N/A case 'q': env.setVerbose(false); break;

77N/A case 'e': env.setGenerateHtml(false); break;

77N/A case 'P': addProjects = true; break;

77N/A case 'p': defaultProject = getopt.getOptarg(); break;

77N/A case 'c': env.setCtags(getopt.getOptarg()); break;

77N/A case 'w': {

77N/A String webapp = getopt.getOptarg();

77N/A if (webapp.startsWith("/") || webapp.startsWith("http")) {

77N/A ;

0N/A } else {

77N/A webapp = "/" + webapp;

77N/A }

77N/A if (webapp.endsWith("/")) {

77N/A env.setUrlPrefix(webapp + "s?");

77N/A } else {

77N/A env.setUrlPrefix(webapp + "/s?");

77N/A }

77N/A }

77N/A break;

77N/A case 'W': configFilename = getopt.getOptarg(); break;

77N/A case 'U': configHost = getopt.getOptarg(); break;

111N/A case 'R':

111N/A // already handled

77N/A break;

77N/A case 'n': runIndex = false; break;

77N/A case 'H': refreshHistory = true; break;

240N/A case 'h' : repositories.add(getopt.getOptarg()); break;

173N/A case 'r': {

173N/A if (getopt.getOptarg().equalsIgnoreCase("on")) {

173N/A env.setRemoteScmSupported(true);

173N/A } else if (getopt.getOptarg().equalsIgnoreCase("off")) {

254N/A env.setRemoteScmSupported(false);

173N/A } else {

173N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -r");

173N/A System.err.println(" Ex: \"-r on\" will allow retrival for remote SCM systems");

254N/A System.err.println(" \"-r off\" will ignore SCM for remote systems");

173N/A }

173N/A }

173N/A break;

253N/A case 'O': {

312N/A boolean oldval = env.isOptimizeDatabase();

253N/A if (getopt.getOptarg().equalsIgnoreCase("on")) {

253N/A env.setOptimizeDatabase(true);

253N/A } else if (getopt.getOptarg().equalsIgnoreCase("off")) {

253N/A env.setOptimizeDatabase(false);

253N/A } else {

253N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -O");

253N/A System.err.println(" Ex: \"-O on\" will optimize the database as part of the index generation");

253N/A System.err.println(" \"-O off\" disable optimization of the index database");

253N/A }

312N/A if (oldval != env.isOptimizeDatabase()) {

312N/A optimizedChanged = true;

312N/A }

253N/A }

253N/A break;

99N/A case 'v': env.setVerbose(true); break;

77N/A

77N/A case 's': {

77N/A File file = new File(getopt.getOptarg());

77N/A if (!file.isDirectory()) {

77N/A System.err.println("ERROR: No such directory: " + file.toString());

0N/A System.exit(1);

0N/A }

77N/A

125N/A env.setSourceRootFile(file);

77N/A break;

77N/A }

205N/A case 'd':

205N/A env.setDataRoot(getopt.getOptarg());

205N/A break;

112N/A case 'i':

112N/A env.getIgnoredNames().add(getopt.getOptarg());

112N/A break;

77N/A case 'S' : searchRepositories = true; break;

106N/A case 'Q' :

119N/A if (getopt.getOptarg().equalsIgnoreCase("on")) {

106N/A env.setQuickContextScan(true);

119N/A } else if (getopt.getOptarg().equalsIgnoreCase("off")) {

106N/A env.setQuickContextScan(false);

119N/A } else {

119N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -Q");

119N/A System.err.println(" Ex: \"-Q on\" will just scan a \"chunk\" of the file and insert \"[..all..]\"");

119N/A System.err.println(" \"-Q off\" will try to build a more accurate list by reading the complete file.");

106N/A }

106N/A

106N/A break;

99N/A case 'm' : {

99N/A try {

99N/A env.setIndexWordLimit(Integer.parseInt(getopt.getOptarg()));

99N/A } catch (NumberFormatException exp) {

99N/A System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());

99N/A System.exit(1);

99N/A }

99N/A break;

99N/A }

125N/A case 'a' :

125N/A if (getopt.getOptarg().equalsIgnoreCase("on")) {

125N/A env.setAllowLeadingWildcard(true);

125N/A } else if (getopt.getOptarg().equalsIgnoreCase("off")) {

125N/A env.setAllowLeadingWildcard(false);

125N/A } else {

125N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -a");

125N/A System.err.println(" Ex: \"-a on\" will allow a search to start with a wildcard");

125N/A System.err.println(" \"-a off\" will disallow a search to start with a wildcard");

126N/A System.exit(1);

125N/A }

125N/A

125N/A break;

126N/A

126N/A case 'A': {

126N/A String[] arg = getopt.getOptarg().split(":");

126N/A if (arg.length != 2) {

126N/A System.err.println("ERROR: You must specify: -A extension:class");

126N/A System.err.println(" Ex: -A foo:org.opensolaris.opengrok.analysis.c.CAnalyzer");

126N/A System.err.println(" will use the C analyzer for all files ending with .foo");

126N/A System.err.println(" Ex: -A c:-");

126N/A System.err.println(" will disable the c-analyzer for for all files ending with .c");

126N/A System.exit(1);

126N/A }

126N/A

126N/A arg[0] = arg[0].substring(arg[0].lastIndexOf('.') + 1).toUpperCase();

126N/A if (arg[1].equals("-")) {

126N/A AnalyzerGuru.addExtension(arg[0], null);

126N/A break;

126N/A }

126N/A

126N/A try {

210N/A AnalyzerGuru.addExtension(

210N/A arg[0],

210N/A AnalyzerGuru.findFactory(arg[1]));

210N/A } catch (Exception e) {

210N/A System.err.println("Unable to use " + arg[1] +

210N/A " as a FileAnalyzerFactory");

210N/A e.printStackTrace();

126N/A System.exit(1);

126N/A }

126N/A }

126N/A break;

144N/A case 'L' :

144N/A env.setWebappLAF(getopt.getOptarg());

144N/A break;

261N/A case 'T' :

261N/A try {

261N/A noThreads = Integer.parseInt(getopt.getOptarg());

261N/A } catch (NumberFormatException exp) {

261N/A System.err.println("ERROR: Failed to parse argument to \"-T\": " + exp.getMessage());

261N/A System.exit(1);

261N/A }

261N/A break;

296N/A case 'l' :

296N/A if (getopt.getOptarg().equalsIgnoreCase("on")) {

296N/A env.setUsingLuceneLocking(true);

296N/A } else if (getopt.getOptarg().equalsIgnoreCase("off")) {

296N/A env.setUsingLuceneLocking(false);

296N/A } else {

296N/A System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -l");

296N/A System.err.println(" Ex: \"-l on\" will enable locks in Lucene");

296N/A System.err.println(" \"-l off\" will disable locks in Lucene");

296N/A }

296N/A break;

293N/A case '?':

293N/A System.err.println(cmdOptions.getUsage());

293N/A System.exit(0);

293N/A break;

293N/A

77N/A default:

293N/A System.err.println("Internal Error - Unimplemented cmdline option: " + (char)cmd);

0N/A System.exit(1);

0N/A }

0N/A }

77N/A

77N/A int optind = getopt.getOptind();

205N/A if (optind != -1) {

77N/A while (optind < argv.length) {

77N/A subFiles.add(argv[optind]);

77N/A ++optind;

77N/A }

77N/A }

58N/A

312N/A getInstance().prepareIndexer(env, searchRepositories, addProjects,

312N/A defaultProject,configFilename,refreshHistory,

312N/A listFiles,createDict,subFiles,repositories);

312N/A if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {

312N/A IndexChangedListener progress = new DefaultIndexChangedListener();

312N/A getInstance().doIndexerExecution(update, noThreads, subFiles,

312N/A progress);

312N/A }

312N/A getInstance().sendToConfigHost(env, configHost);

312N/A } catch (IndexerException ex) {

428N/A OpenGrokLogger.getLogger().log(Level.SEVERE, "Exception running indexer", ex);

312N/A System.err.println(cmdOptions.getUsage());

312N/A System.exit(1);

312N/A } catch (IOException ioe) {

312N/A System.err.println("Got IOException " + ioe);

428N/A OpenGrokLogger.getLogger().log(Level.SEVERE, "Exception running indexer", ioe);

312N/A System.exit(1);

312N/A }

312N/A }

312N/A

312N/A }

312N/A

312N/A

312N/A

312N/A public void prepareIndexer(RuntimeEnvironment env,

312N/A boolean searchRepositories,

312N/A boolean addProjects,

312N/A String defaultProject,

312N/A String configFilename,

312N/A boolean refreshHistory,

312N/A boolean listFiles,

312N/A boolean createDict,

312N/A ArrayList<String> subFiles,

312N/A ArrayList<String> repositories) throws IndexerException,IOException {

312N/A

58N/A if (env.getDataRootPath() == null) {

312N/A throw new IndexerException("ERROR: Please specify a DATA ROOT path");

0N/A }

77N/A

58N/A if (env.getSourceRootFile() == null) {

58N/A File srcConfig = new File(env.getDataRootPath(), "SRC_ROOT");

58N/A String line = null;

0N/A if(srcConfig.exists()) {

0N/A try {

0N/A BufferedReader sr = new BufferedReader(new FileReader(srcConfig));

58N/A line = sr.readLine();

0N/A sr.close();

0N/A } catch (IOException e) {

0N/A }

0N/A }

58N/A if(line == null) {

312N/A throw new IndexerException("ERROR: please specify a SRC_ROOT with option -s !");

0N/A }

58N/A env.setSourceRoot(line);

77N/A

58N/A if (!env.getSourceRootFile().isDirectory()) {

312N/A throw new IndexerException("ERROR: No such directory:" + line);

0N/A }

0N/A }

77N/A

99N/A if (!env.validateExuberantCtags()) {

410N/A throw new IndexerException("Didn't find Exuberant Ctags");

0N/A }

77N/A

11N/A if (searchRepositories) {

99N/A if (env.isVerbose()) {

99N/A System.out.println("Scanning for repositories...");

99N/A }

99N/A env.getRepositories().clear();

11N/A long start = System.currentTimeMillis();

290N/A HistoryGuru.getInstance().addRepositories(env.getSourceRootPath());

11N/A long time = (System.currentTimeMillis() - start) / 1000;

99N/A if (env.isVerbose()) {

99N/A System.out.println("Done searching for repositories (" + time + "s)");

99N/A }

11N/A }

77N/A

58N/A if (addProjects) {

77N/A File files[] = env.getSourceRootFile().listFiles();

70N/A List<Project> projects = env.getProjects();

70N/A projects.clear();

58N/A for (File file : files) {

77N/A if (!file.getName().startsWith(".") && file.isDirectory()) {

70N/A projects.add(new Project(file.getName(), "/" + file.getName()));

58N/A }

58N/A }

77N/A

77N/A // The projects should be sorted...

99N/A Collections.sort(projects, new Comparator<Project>() {

99N/A public int compare(Project p1, Project p2){

99N/A String s1 = p1.getDescription();

99N/A String s2 = p2.getDescription();

77N/A

77N/A int ret;

77N/A if (s1 == null) {

77N/A ret = (s2 == null) ? 0 : 1;

77N/A } else {

77N/A ret = s1.compareTo(s2);

77N/A }

77N/A return ret;

77N/A }

77N/A });

58N/A }

77N/A

77N/A if (defaultProject != null) {

77N/A for (Project p : env.getProjects()) {

77N/A if (p.getPath().equals(defaultProject)) {

77N/A env.setDefaultProject(p);

77N/A break;

77N/A }

77N/A }

77N/A }

99N/A

58N/A if (configFilename != null) {

99N/A if (env.isVerbose()) {

99N/A System.out.println("Writing configuration to " + configFilename);

99N/A System.out.flush();

99N/A }

58N/A env.writeConfiguration(new File(configFilename));

99N/A if (env.isVerbose()) {

99N/A System.out.println("Done...");

99N/A System.out.flush();

99N/A }

58N/A }

112N/A

58N/A if (refreshHistory) {

203N/A HistoryGuru.getInstance().createCache();

352N/A } else if (repositories != null && repositories.size() > 0) {

240N/A HistoryGuru.getInstance().createCache(repositories);

58N/A }

58N/A

207N/A if (listFiles) {

207N/A IndexDatabase.listAllFiles(subFiles);

207N/A }

207N/A

207N/A if (createDict) {

207N/A IndexDatabase.listFrequentTokens(subFiles);

207N/A }

312N/A }

312N/A

312N/A

312N/A public void doIndexerExecution(final boolean update, int noThreads, List<String> subFiles,

312N/A IndexChangedListener progress)

312N/A throws IOException {

312N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();

312N/A env.register();

312N/A log.info("Starting indexExecution");

207N/A

270N/A ExecutorService executor = Executors.newFixedThreadPool(noThreads);

312N/A

352N/A if (subFiles == null || subFiles.isEmpty()) {

312N/A if (update) {

264N/A IndexDatabase.updateAll(executor, progress);

270N/A } else if (env.isOptimizeDatabase()) {

270N/A IndexDatabase.optimizeAll(executor);

270N/A }

270N/A } else {

274N/A List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();

274N/A

270N/A for (String path : subFiles) {

270N/A Project project = Project.getProject(path);

297N/A if (project == null && env.hasProjects()) {

270N/A System.err.println("Warning: Could not find a project for \"" + path + "\"");

270N/A } else {

297N/A IndexDatabase db;

297N/A if (project != null) {

297N/A db = new IndexDatabase(project);

297N/A } else {

297N/A db = new IndexDatabase();

297N/A }

274N/A int idx = dbs.indexOf(db);

274N/A if (idx != -1) {

274N/A db = dbs.get(idx);

274N/A }

274N/A

274N/A if (db.addDirectory(path)) {

274N/A if (idx == -1) {

274N/A dbs.add(db);

274N/A }

274N/A } else {

274N/A System.err.println("Warning: Directory does not exist \"" + path + "\"");

274N/A }

274N/A }

274N/A }

274N/A

274N/A for (final IndexDatabase db : dbs) {

270N/A final boolean optimize = env.isOptimizeDatabase();

270N/A db.addIndexChangedListener(progress);

270N/A executor.submit(new Runnable() {

261N/A

270N/A public void run() {

270N/A try {

270N/A if (update) {

264N/A db.update();

270N/A } else if (optimize) {

270N/A db.optimize();

261N/A }

270N/A } catch (Exception e) {

270N/A e.printStackTrace();

264N/A }

270N/A }

274N/A });

270N/A }

270N/A }

270N/A

270N/A executor.shutdown();

270N/A while (!executor.isTerminated()) {

270N/A try {

270N/A Thread.sleep(1000);

270N/A } catch (Exception e) {

270N/A

264N/A }

264N/A }

312N/A }

58N/A

312N/A

312N/A public void sendToConfigHost(RuntimeEnvironment env, String configHost) {

58N/A if (configHost != null) {

58N/A String[] cfg = configHost.split(":");

99N/A if (env.isVerbose()) {

312N/A log.info("Send configuration to: " + configHost);

99N/A }

58N/A

58N/A if (cfg.length == 2) {

58N/A try {

58N/A InetAddress host = InetAddress.getByName(cfg[0]);

58N/A RuntimeEnvironment.getInstance().writeConfiguration(host, Integer.parseInt(cfg[1]));

58N/A } catch (Exception ex) {

312N/A log.log(Level.SEVERE,"Failed to send configuration to "

312N/A + configHost,ex);

58N/A }

58N/A } else {

58N/A System.err.println("Syntax error: ");

58N/A for (String s : cfg) {

58N/A System.err.print("[" + s + "]");

58N/A }

58N/A System.err.println();

58N/A }

99N/A if (env.isVerbose()) {

312N/A log.info("Configuration successfully updated");

99N/A }

312N/A }

0N/A }

260N/A

260N/A private Indexer() {

260N/A }

0N/A}