/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * See LICENSE.txt included in this distribution for the specific * language governing permissions and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at LICENSE.txt. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.history; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import org.opensolaris.opengrok.configuration.Configuration; import org.opensolaris.opengrok.configuration.RuntimeEnvironment; import org.opensolaris.opengrok.index.IgnoredNames; /** * The HistoryGuru is used to implement an transparent layer to the various * source control systems. * * @author Chandan */ public final class HistoryGuru { private static final Logger logger = Logger.getLogger(HistoryGuru.class.getName()); /** The one and only instance of the HistoryGuru */ private static HistoryGuru instance = new HistoryGuru(); /** The history cache to use */ private final HistoryCache historyCache; private Map repositories = new HashMap(); private final int scanningDepth; /** * Creates a new instance of HistoryGuru, and try to set the default * source control system. */ private HistoryGuru() { HistoryCache cache = null; Configuration cfg = RuntimeEnvironment.getConfig(); scanningDepth = cfg.getScanningDepth(); if (cfg.isHistoryCache()) { if (cfg.isHistoryCacheInDB()) { cache = new JDBCHistoryCache(); } else { cache = new FileHistoryCache(); } try { cache.initialize(); } catch (HistoryException he) { logger.warning("Failed to initialize the history cache: " + he.getMessage()); logger.log(Level.FINE, "HistoryGuru", he); // Failed to initialize, run without a history cache cache = null; } } historyCache = cache; } /** * Get the one and only instance of the HistoryGuru * @return the one and only HistoryGuru instance */ public static HistoryGuru getInstance() { return instance; } /** * Check whether a cache should be used for the history log. * @return {@code true} if the history cache has been enabled and is ready * for use. */ private boolean useCache() { return historyCache != null; } /** * Get a string with information about the history cache. * * @return a free form text string describing the history cache instance * @throws HistoryException if an error occurred while getting the info */ public String getCacheInfo() throws HistoryException { return useCache() ? "No cache" : historyCache.getInfo(); } /** * Annotate the specified revision of a file. * * @param file the file to annotate * @param rev the revision to annotate (null means BASE) * @return file annotation, or null if the * HistoryParser does not support annotation * @throws IOException */ public Annotation annotate(File file, String rev) throws IOException { Annotation ret = null; Repository repos = getRepository(file); if (repos != null) { ret = repos.annotate(file, rev); History hist = null; try { hist = repos.getHistory(file); } catch (HistoryException ex) { logger.log(Level.FINEST, "Cannot get messages for tooltip", ex); } if (hist != null && ret != null) { Set revs = ret.getRevisions(); // !!! cannot do this because of not matching rev ids (keys) // first is the most recent one, so we need the position of "rev" // until the end of the list //if (hent.indexOf(rev)>0) { // hent = hent.subList(hent.indexOf(rev), hent.size()); //} for (HistoryEntry he : hist.getHistoryEntries()) { String cmr = he.getRevision(); //TODO this is only for mercurial, for other SCMs it might also // be a problem, we need to revise how we shorten the rev # for // annotate String[] brev = cmr.split(":"); if (revs.contains(brev[0])) { ret.addDesc(brev[0], he.getRevision(), he.getMessage(), he.getAuthor(), he.getDate()); } } } } return ret; } /** * Get the appropriate history reader for the file specified by parent and * basename. * * @param file The file to get the history reader for (canonical path incl. * source root). * @throws HistoryException If an error occurs while getting the history * @return A HistorReader that may be used to read out history data for a * named file */ @SuppressWarnings("resource") public HistoryReader getHistoryReader(File file) throws HistoryException { History history = getHistory(file, false); return history == null ? null : new HistoryReader(history); } /** * Get the history for the specified file. The type of the returned history * (directory or file) gets automatically determined: If file physically * exists and is a directory, directory is assumed, file otherwise. * * @param path The file to get the history for (canonical path incl. * source root). * @return The history for the given path including a list of files touched * by each changeset. * @throws HistoryException on error when accessing the history */ public History getHistory(File path) throws HistoryException { return getHistory(path, true); } /** * Get the history for the specified file. The type of the returned history * (directory or file) gets automatically determined: If file physically * exists and is a directory, directory is assumed, file otherwise. * * @param path The file to get the history for (canonical path incl. * source root). * @param withFiles If {@code true} the returned history will contain * a list of files touched by each changeset (the file list may be * skipped if {@code false}, but it doesn't have to) * @return The history for the given path. * @throws HistoryException on error when accessing the history */ public History getHistory(File path, boolean withFiles) throws HistoryException { return getHistory(path, withFiles, null); } /** * Get the history for the specified path. The type of the returned history * (directory or file) gets automatically determined: If path * physically exists and is a directory, directory is assumed, file otherwise. * * @param path The path name to get the history for (canonical path * incl. source root). * @param withFiles If {@code true} the returned history will contain * a list of files touched by each changeset (the file list may be * skipped if {@code false}, but it doesn't have to) * @param isDir If {@code null} it behaves like * {@link #getHistory(File, boolean)} (auto detect history type). * If {@code true} doesn't check the file and blindly assumes type * directory. Otherwise a file history gets returned. * @return the history for the given path * @throws HistoryException on error when accessing the history * @see #isDirectory(String) */ public History getHistory(File path, boolean withFiles, Boolean isDir) throws HistoryException { File dir = path; if (isDir == null) { if (!path.isDirectory()) { dir = path.getParentFile(); isDir = Boolean.FALSE; } else { isDir = Boolean.TRUE; } } else if (!isDir.booleanValue()) { dir = path.getParentFile(); } final Repository repos = getRepository(dir); History history = null; if (repos != null && repos.isWorking() && repos.fileHasHistory(path) && (!repos.isRemote() || RuntimeEnvironment.getConfig() .isRemoteScmSupported())) { if (useCache() && historyCache.supportsRepository(repos)) { history = historyCache.get(path, repos, withFiles, isDir); } else { history = repos.getHistory(path); } } return history; } /** * Get a named revision of the specified file. * @param parent The directory (canonical path incl. source root) containing * the file. * @param basename The name of the file. * @param rev The revision to get. * @return An InputStream containing the named revision of the file. * @see "Repository.getHistoryGet(String, String, String)" */ public InputStream getRevision(String parent, String basename, String rev) { InputStream ret = null; Repository rep = getRepository(new File(parent)); if (rep != null) { ret = rep.getHistoryGet(parent, basename, rev); } return ret; } /** * Does this directory contain files with source control information? * In contrast to {@link #isDirectory(String)} this usually doesn't hit the * history DB but solely ask the repository. * * @param file The name of the directory (canonical path incl. source root). * @return {@code true} if the files in this directory have associated * revision history */ public boolean hasHistory(File file) { Repository repos = getRepository(file); return repos == null ? false : repos.isWorking() && repos.fileHasHistory(file) && (RuntimeEnvironment.getConfig().isRemoteScmSupported() || !repos.isRemote()); } /** * Check if we can annotate the specified file. Doesn't hit the history DB * but asks the repository. * * @param file the file to check (canonical path incl. source root). * @return {@code true} if the file is under version control and the * version control system supports annotation. */ public boolean hasAnnotation(File file) { if (!file.isDirectory()) { Repository repos = getRepository(file); if (repos != null && repos.isWorking()) { return repos.fileHasAnnotation(file); } } return false; } /** * Get the last modified times for all files and subdirectories excluding * '.' and '..' in the specified directory the history cache has information * about. The returned map is keyed by the name of the file or subdirectory * and does neither contain a {@code null} key nor a {@code null} value. * * @param directory the directory whose files to check (canonical path incl. * source root). * @param path2rev If not {@code null}, pathes including their latest * revisions get stored into this map. * @return a possible empty map. */ public Map getLastModifiedTimes(File directory, Map path2rev) { if (!useCache()) { return Collections.emptyMap(); } // if source root if (directory.equals(RuntimeEnvironment.getConfig().getSourceRootFile())) { try { Map result = historyCache.getLastModifiedTimes(path2rev); return result; } catch (HistoryException e) { logger.warning(e.getLocalizedMessage()); if (logger.isLoggable(Level.FINE)) { logger.log(Level.FINE, "getLastModifiedTimes()", e); } } return Collections.emptyMap(); } // else try repo Repository repository = getRepository(directory); if (repository != null) { try { Map result = historyCache .getLastModifiedTimes(directory, repository, path2rev); return result; } catch (HistoryException e) { logger.warning(e.getLocalizedMessage()); if (logger.isLoggable(Level.FINE)) { logger.log(Level.FINE, "getLastModifiedTimes()", e); } } } return Collections.emptyMap(); } private void addRepositories(File[] files, Collection repos, IgnoredNames ignoredNames, int depth) { addRepositories(files, repos, ignoredNames, true, depth); } /** * recursivelly search for repositories with a depth limit * @param files list of files to check if they contain a repo * @param repos list of found repos * @param ignoredNames what files to ignore * @param recursiveSearch whether to use recursive search * @param depth current depth - using global scanningDepth - one can limit * this to improve scanning performance */ private void addRepositories(File[] files, Collection repos, IgnoredNames ignoredNames, boolean recursiveSearch, int depth) { for (File file : files) { Repository repository = null; try { repository = RepositoryFactory.getRepository(file); } catch (InstantiationException ie) { logger.warning("Could not create repoitory for '" + file + "', could not instantiate the repository: " + ie.getMessage()); logger.log(Level.FINE, "addRepositories", ie); } catch (IllegalAccessException iae) { logger.warning("Could not create repoitory for '" + file + "', missing access rights: " + iae.getMessage()); } if (repository == null) { // Not a repository, search it's sub-dirs if (file.isDirectory() && !ignoredNames.match(file)) { File subFiles[] = file.listFiles(); if (subFiles == null) { logger.warning("Failed to get sub directories for '" + file.getAbsolutePath() + "', check access permissions."); } else if (depth<=scanningDepth) { addRepositories(subFiles, repos, ignoredNames, depth+1); } } } else { try { String path = file.getCanonicalPath(); repository.setDirectoryName(path); if (RuntimeEnvironment.getConfig().isVerbose()) { logger.log(Level.FINE, "Adding <{0}> repository: <{1}>", new Object[]{repository.getClass().getName(), path}); } repos.add(new RepositoryInfo(repository)); // @TODO: Search only for one type of repository - the one found here if (recursiveSearch && repository.supportsSubRepositories()) { File subFiles[] = file.listFiles(); if (subFiles == null) { logger.warning("Failed to get sub directories for '" + file.getAbsolutePath() + "', check access permissions."); } else if (depth<=scanningDepth) { // Search only one level down - if not: too much // stat'ing for huge Mercurial repositories addRepositories(subFiles, repos, ignoredNames, false, depth+1); } } } catch (IOException exp) { logger.warning("Failed to get canonical path for '" + file.getAbsolutePath() + "'. Repository will be ignored: " + exp.getMessage()); logger.log(Level.FINE, "addRepositories", exp); } } } } /** * Search through the all of the directories and add all of the source * repositories found. * * @param dir the root directory to start the search in. */ public void addRepositories(String dir) { List repos = new ArrayList(); addRepositories(new File[] { new File(dir) }, repos, RuntimeEnvironment.getConfig().getIgnoredNames(), 0); RuntimeEnvironment.getConfig().setRepositories(repos); invalidateRepositories(repos); } /** * Update the source the contents in the source repositories. */ public void updateRepositories() { boolean verbose = RuntimeEnvironment.getConfig().isVerbose(); for (Map.Entry entry : repositories.entrySet()) { Repository repository = entry.getValue(); String path = entry.getKey(); String type = repository.getClass().getSimpleName(); if (repository.isWorking()) { if (verbose) { logger.log(Level.INFO, "Update {0} repository in ''{1}''", new String[] { type, path }); } try { repository.update(); } catch (UnsupportedOperationException e) { logger.warning("Skipping update of " + type + " repository" + " in '" + path + "': Not implemented"); } catch (Exception e) { logger.warning("An error occured while updating '" + path + "' (" + type + "): " + e.getMessage()); logger.log(Level.FINE, "updateRepositories", e); } } else { logger.warning("Skipping update of " + type + " repository in '" + path + "': Missing SCM dependencies?"); } } } /** * Update the source the contents in the source repositories. * @param paths A list of files/directories to update */ public void updateRepositories(Collection paths) { boolean verbose = RuntimeEnvironment.getConfig().isVerbose(); List repos = getReposFromString(paths); for (Repository repository : repos) { String type = repository.getClass().getSimpleName(); if (repository.isWorking()) { if (verbose) { logger.log(Level.INFO, "Update {0} repository in ''{1}''", new String[] { type, repository.getDirectoryName()}); } try { repository.update(); } catch (UnsupportedOperationException e) { logger.warning("Skipping update of " + type + " repository" + " in '" + repository.getDirectoryName() + "': Not implemented"); } catch (Exception e) { logger.warning("An error occured while updating '" + repository.getDirectoryName() + "' (" + type + "): " + e.getMessage()); logger.log(Level.FINE, "updateRepositories", e); } } else { logger.warning("Skipping update of " + type + " repository in '" + repository.getDirectoryName() + "': Missing SCM dependencies?"); } } } private void createCache(Repository repository, String sinceRevision) { if (!useCache()) { return; } String path = repository.getDirectoryName(); String type = repository.getClass().getSimpleName(); if (repository.isWorking()) { boolean verbose = RuntimeEnvironment.getConfig().isVerbose(); long start = System.currentTimeMillis(); if (verbose) { logger.log(Level.INFO, "Create historycache for ''{0}'' ({1})", new String[]{path, type}); } try { repository.createCache(historyCache, sinceRevision); } catch (Exception e) { logger.warning("An error occured while creating cache for '" + path + "' (" + type + "): " + e.getMessage()); logger.log(Level.FINE, "createCache", e); } if (verbose) { long stop = System.currentTimeMillis(); logger.log(Level.INFO, "Creating historycache for ''{0}'' took {1}ms", new String[]{path, String.valueOf(stop - start)}); } } else { logger.warning("Skipping creation of historycache of " + type + " repository in '" + path + "': Missing SCM dependencies?"); } } /** * Create the history cache for the given repositories, if caching is * enabled and the repository in question supports history for directories. * @param allRepos repositories in question. If {@code null} all registered * repos are used instead. */ public void createCache(Collection allRepos) { if (!useCache()) { return; } Collection repositories = allRepos == null ? this.repositories.values() : getReposFromString(allRepos); int num = Runtime.getRuntime().availableProcessors() * 2; String total = System.getProperty(Configuration.PROPERTY_KEY_PREFIX + "history.NumCacheThreads"); if (total != null) { try { num = Integer.parseInt(total, 10); } catch (Throwable t) { logger.warning("Failed to parse the number of cache threads to use for cache creation: " + t.getMessage()); logger.log(Level.FINE, "createCacheReal", t); } } ExecutorService executor = Executors.newFixedThreadPool(num); for (final Repository repos : repositories) { final String latestRev; try { latestRev = historyCache.getLatestCachedRevision(repos); } catch (HistoryException he) { logger.warning("Failed to retrieve latest cached revision for '" + repos.getDirectoryName() + "': " + he.getMessage()); logger.log(Level.FINE, "createCacheReal", he); continue; } executor.submit(new Runnable() { @SuppressWarnings("synthetic-access") @Override public void run() { createCache(repos, latestRev); } }); } executor.shutdown(); while (!executor.isTerminated()) { try { // Wait forever executor.awaitTermination(999, TimeUnit.DAYS); } catch (InterruptedException exp) { logger.warning("Received interrupt while waiting for executor to finish: " + exp.getMessage()); logger.log(Level.FINE, "createCacheReal", exp); } } // The cache has been populated. Now, optimize how it is stored on // disk to enhance performance and save space. try { historyCache.optimize(); } catch (HistoryException he) { logger.warning("Failed optimizing the history cache database: " + he.getMessage()); logger.log(Level.FINE, "createCacheReal", he); } } /** * Remove the history cache for the given repositories. * @param repositories repositories in question. * @throws HistoryException */ public void removeCache(Collection repositories) throws HistoryException { List repos = getReposFromString(repositories); HistoryCache cache = historyCache; if (cache == null) { if (RuntimeEnvironment.getConfig().isHistoryCacheInDB()) { cache = new JDBCHistoryCache(); cache.initialize(); } else { cache = new FileHistoryCache(); } } for (Repository r : repos) { try { cache.clear(r); logger.info("History cache for '" + r.getDirectoryName() + "' cleared."); } catch (HistoryException e) { logger.warning("Clearing history cache for repository '" + r.getDirectoryName() + "' failed: " + e.getLocalizedMessage()); } } invalidateRepositories(repos); } /** * Create the history cache for all of the repositories. */ public void createCache() { createCache(null); } private List getReposFromString(Collection repositories) { ArrayList repos = new ArrayList(); File root = RuntimeEnvironment.getConfig().getSourceRootFile(); for (String file : repositories) { File f = new File(root, file); Repository r = getRepository(f); if (r == null) { logger.warning("Could not locate a repository for '" + f.getAbsolutePath() + "'"); } else if (!repos.contains(r)){ repos.add(r); } } return repos; } /** * Check, whether the given path denotes a directory in the history cache. * This really hits the DB, so caching is advised. * * @param path The canonical path to the file in question, * e.g. sourceRoot + '/' + projectdir + '/' + subdir with '/' * path separators, only! One trailing slash gets automatically removed * if it exists. * @return {@code null} if an error occured, {@code Boolean#TRUE} if there * is a directory with the given name in the history cache, * {@link Boolean#FALSE} otherwise. */ public Boolean isDirectory(String path) { if (!useCache() || path == null) { return null; } File dir = new File(path); Repository repo = getRepository(dir); try { return Boolean.valueOf(historyCache.hasCacheForDirectory(dir, repo)); } catch (HistoryException e) { logger.warning(e.getLocalizedMessage()); logger.log(Level.FINE, "isDirectory", e); return null; } } /** * Ensure that we have a directory in the cache. If it's not there, fetch * its history and populate the cache. If it's already there, and the * cache is able to tell how recent it is, attempt to update it to the * most recent revision. * * @param file the root path to test (canonical path incl. source root). * @throws HistoryException if an error occurs while accessing the * history cache */ public void ensureHistoryCacheExists(File file) throws HistoryException { if (!useCache()) { return; } Repository repository = getRepository(file); if (repository == null) { // no repository -> no history :( return; } String sinceRevision = null; if (historyCache.hasCacheForDirectory(file, repository)) { sinceRevision = historyCache.getLatestCachedRevision(repository); if (sinceRevision == null) { // Cache already exists, but we don't know how recent it is, // so don't do anything. return; } } // Create cache from the beginning if it doesn't exist, or update it // incrementally otherwise. createCache(getRepository(file), sinceRevision); } /** * Get the with this instance registered repository for the given file. * @param path canonical path incl. source root to the source file in * question. * @return {@code null} if unknown, the related repository otherwise. */ protected Repository getRepository(File path) { Map repos = repositories; File file = path; try { file = path.getCanonicalFile(); } catch (IOException e) { logger.warning("Failed to get canonical path for '" + path + "': " + e.getLocalizedMessage()); return null; } while (file != null) { Repository r = repos.get(file.getAbsolutePath()); if (r != null) { return r; } file = file.getParentFile(); } return null; } /** * Invalidate the current list of known repositories! * * @param repos The new repositories */ public void invalidateRepositories(Collection repos) { if (repos == null || repos.isEmpty()) { repositories.clear(); } else { Map nrep = new HashMap(repos.size()); for (RepositoryInfo i : repos) { try { Repository r = RepositoryFactory.getRepository(i); if (r == null) { logger.warning("Failed to instanciate internal repository data for " + i.getType() + " in '" + i.getDirectoryName() + "'"); } else { nrep.put(r.getDirectoryName(), r); } } catch (InstantiationException ex) { logger.warning("Could not create " + i.getType() + " for '" + i.getDirectoryName() + "', could not instantiate the repository: " + ex.getMessage()); logger.log(Level.FINE, "invalidateRepositories", ex); } catch (IllegalAccessException iae) { logger.warning("Could not create " + i.getType() + " for '" + i.getDirectoryName() + "', missing access rights: " + iae.getMessage()); logger.log(Level.FINE, "invalidateRepositories", iae); } } repositories = nrep; } } }