/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
*/
package org.opensolaris.opengrok.history;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.opensolaris.opengrok.configuration.Configuration;
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
import org.opensolaris.opengrok.index.IgnoredNames;
/**
* The HistoryGuru is used to implement an transparent layer to the various
* source control systems.
*
* @author Chandan
*/
public final class HistoryGuru {
private static final Logger logger =
Logger.getLogger(HistoryGuru.class.getName());
/** The one and only instance of the HistoryGuru */
private static HistoryGuru instance = new HistoryGuru();
/** The history cache to use */
private final HistoryCache historyCache;
private Map<String, Repository> repositories =
new HashMap<String, Repository>();
private final int scanningDepth;
/**
* Creates a new instance of HistoryGuru, and try to set the default
* source control system.
*/
private HistoryGuru() {
HistoryCache cache = null;
Configuration cfg = RuntimeEnvironment.getConfig();
scanningDepth = cfg.getScanningDepth();
if (cfg.isHistoryCache()) {
if (cfg.isHistoryCacheInDB()) {
cache = new JDBCHistoryCache();
} else {
cache = new FileHistoryCache();
}
try {
cache.initialize();
} catch (HistoryException he) {
logger.warning("Failed to initialize the history cache: "
+ he.getMessage());
logger.log(Level.FINE, "HistoryGuru", he);
// Failed to initialize, run without a history cache
cache = null;
}
}
historyCache = cache;
}
/**
* Get the one and only instance of the HistoryGuru
* @return the one and only HistoryGuru instance
*/
public static HistoryGuru getInstance() {
return instance;
}
/**
* Check whether a cache should be used for the history log.
* @return {@code true} if the history cache has been enabled and is ready
* for use.
*/
private boolean useCache() {
return historyCache != null;
}
/**
* Get a string with information about the history cache.
*
* @return a free form text string describing the history cache instance
* @throws HistoryException if an error occurred while getting the info
*/
public String getCacheInfo() throws HistoryException {
return useCache() ? "No cache" : historyCache.getInfo();
}
/**
* Annotate the specified revision of a file.
*
* @param file the file to annotate
* @param rev the revision to annotate (<code>null</code> means BASE)
* @return file annotation, or <code>null</code> if the
* <code>HistoryParser</code> does not support annotation
* @throws IOException
*/
public Annotation annotate(File file, String rev) throws IOException {
Annotation ret = null;
Repository repos = getRepository(file);
if (repos != null) {
ret = repos.annotate(file, rev);
History hist = null;
try {
hist = repos.getHistory(file);
} catch (HistoryException ex) {
logger.log(Level.FINEST, "Cannot get messages for tooltip", ex);
}
if (hist != null && ret != null) {
Set<String> revs = ret.getRevisions();
// !!! cannot do this because of not matching rev ids (keys)
// first is the most recent one, so we need the position of "rev"
// until the end of the list
//if (hent.indexOf(rev)>0) {
// hent = hent.subList(hent.indexOf(rev), hent.size());
//}
for (HistoryEntry he : hist.getHistoryEntries()) {
String cmr = he.getRevision();
//TODO this is only for mercurial, for other SCMs it might also
// be a problem, we need to revise how we shorten the rev # for
// annotate
String[] brev = cmr.split(":");
if (revs.contains(brev[0])) {
ret.addDesc(brev[0], he.getRevision(), he.getMessage(),
he.getAuthor(), he.getDate());
}
}
}
}
return ret;
}
/**
* Get the appropriate history reader for the file specified by parent and
* basename.
*
* @param file The file to get the history reader for (canonical path incl.
* source root).
* @throws HistoryException If an error occurs while getting the history
* @return A HistorReader that may be used to read out history data for a
* named file
*/
@SuppressWarnings("resource")
public HistoryReader getHistoryReader(File file) throws HistoryException {
History history = getHistory(file, false);
return history == null ? null : new HistoryReader(history);
}
/**
* Get the history for the specified file. The type of the returned history
* (directory or file) gets automatically determined: If file physically
* exists and is a directory, directory is assumed, file otherwise.
*
* @param path The file to get the history for (canonical path incl.
* source root).
* @return The history for the given path including a list of files touched
* by each changeset.
* @throws HistoryException on error when accessing the history
*/
public History getHistory(File path) throws HistoryException {
return getHistory(path, true);
}
/**
* Get the history for the specified file. The type of the returned history
* (directory or file) gets automatically determined: If file physically
* exists and is a directory, directory is assumed, file otherwise.
*
* @param path The file to get the history for (canonical path incl.
* source root).
* @param withFiles If {@code true} the returned history will contain
* a list of files touched by each changeset (the file list may be
* skipped if {@code false}, but it doesn't have to)
* @return The history for the given path.
* @throws HistoryException on error when accessing the history
*/
public History getHistory(File path, boolean withFiles)
throws HistoryException
{
return getHistory(path, withFiles, null);
}
/**
* Get the history for the specified path. The type of the returned history
* (directory or file) gets automatically determined: If <var>path</var>
* physically exists and is a directory, directory is assumed, file otherwise.
*
* @param path The path name to get the history for (canonical path
* incl. source root).
* @param withFiles If {@code true} the returned history will contain
* a list of files touched by each changeset (the file list may be
* skipped if {@code false}, but it doesn't have to)
* @param isDir If {@code null} it behaves like
* {@link #getHistory(File, boolean)} (auto detect history type).
* If {@code true} doesn't check the file and blindly assumes type
* directory. Otherwise a file history gets returned.
* @return the history for the given path
* @throws HistoryException on error when accessing the history
* @see #isDirectory(String)
*/
public History getHistory(File path, boolean withFiles, Boolean isDir)
throws HistoryException
{
File dir = path;
if (isDir == null) {
if (!path.isDirectory()) {
dir = path.getParentFile();
isDir = Boolean.FALSE;
} else {
isDir = Boolean.TRUE;
}
} else if (!isDir.booleanValue()) {
dir = path.getParentFile();
}
final Repository repos = getRepository(dir);
History history = null;
if (repos != null && repos.isWorking() && repos.fileHasHistory(path)
&& (!repos.isRemote() || RuntimeEnvironment.getConfig()
.isRemoteScmSupported()))
{
if (useCache() && historyCache.supportsRepository(repos)) {
history = historyCache.get(path, repos, withFiles, isDir);
} else {
history = repos.getHistory(path);
}
}
return history;
}
/**
* Get a named revision of the specified file.
* @param parent The directory (canonical path incl. source root) containing
* the file.
* @param basename The name of the file.
* @param rev The revision to get.
* @return An InputStream containing the named revision of the file.
* @see "Repository.getHistoryGet(String, String, String)"
*/
public InputStream getRevision(String parent, String basename, String rev)
{
InputStream ret = null;
Repository rep = getRepository(new File(parent));
if (rep != null) {
ret = rep.getHistoryGet(parent, basename, rev);
}
return ret;
}
/**
* Does this directory contain files with source control information?
* In contrast to {@link #isDirectory(String)} this usually doesn't hit the
* history DB but solely ask the repository.
*
* @param file The name of the directory (canonical path incl. source root).
* @return {@code true} if the files in this directory have associated
* revision history
*/
public boolean hasHistory(File file) {
Repository repos = getRepository(file);
return repos == null
? false
: repos.isWorking() && repos.fileHasHistory(file)
&& (RuntimeEnvironment.getConfig().isRemoteScmSupported()
|| !repos.isRemote());
}
/**
* Check if we can annotate the specified file. Doesn't hit the history DB
* but asks the repository.
*
* @param file the file to check (canonical path incl. source root).
* @return {@code true} if the file is under version control and the
* version control system supports annotation.
*/
public boolean hasAnnotation(File file) {
if (!file.isDirectory()) {
Repository repos = getRepository(file);
if (repos != null && repos.isWorking()) {
return repos.fileHasAnnotation(file);
}
}
return false;
}
/**
* Get the last modified times for all files and subdirectories excluding
* '.' and '..' in the specified directory the history cache has information
* about. The returned map is keyed by the name of the file or subdirectory
* and does neither contain a {@code null} key nor a {@code null} value.
*
* @param directory the directory whose files to check (canonical path incl.
* source root).
* @param path2rev If not {@code null}, pathes including their latest
* revisions get stored into this map.
* @return a possible empty map.
*/
public Map<String, Date> getLastModifiedTimes(File directory,
Map<String, String> path2rev)
{
if (!useCache()) {
return Collections.emptyMap();
}
// if source root
if (directory.equals(RuntimeEnvironment.getConfig().getSourceRootFile())) {
try {
Map<String, Date> result =
historyCache.getLastModifiedTimes(path2rev);
return result;
} catch (HistoryException e) {
logger.warning(e.getLocalizedMessage());
if (logger.isLoggable(Level.FINE)) {
logger.log(Level.FINE, "getLastModifiedTimes()", e);
}
}
return Collections.emptyMap();
}
// else try repo
Repository repository = getRepository(directory);
if (repository != null) {
try {
Map<String, Date> result = historyCache
.getLastModifiedTimes(directory, repository, path2rev);
return result;
} catch (HistoryException e) {
logger.warning(e.getLocalizedMessage());
if (logger.isLoggable(Level.FINE)) {
logger.log(Level.FINE, "getLastModifiedTimes()", e);
}
}
}
return Collections.emptyMap();
}
private void addRepositories(File[] files, Collection<RepositoryInfo> repos,
IgnoredNames ignoredNames, int depth)
{
addRepositories(files, repos, ignoredNames, true, depth);
}
/**
* recursivelly search for repositories with a depth limit
* @param files list of files to check if they contain a repo
* @param repos list of found repos
* @param ignoredNames what files to ignore
* @param recursiveSearch whether to use recursive search
* @param depth current depth - using global scanningDepth - one can limit
* this to improve scanning performance
*/
private void addRepositories(File[] files, Collection<RepositoryInfo> repos,
IgnoredNames ignoredNames, boolean recursiveSearch, int depth) {
for (File file : files) {
Repository repository = null;
try {
repository = RepositoryFactory.getRepository(file);
} catch (InstantiationException ie) {
logger.warning("Could not create repoitory for '"
+ file + "', could not instantiate the repository: "
+ ie.getMessage());
logger.log(Level.FINE, "addRepositories", ie);
} catch (IllegalAccessException iae) {
logger.warning("Could not create repoitory for '"
+ file + "', missing access rights: " + iae.getMessage());
}
if (repository == null) {
// Not a repository, search it's sub-dirs
if (file.isDirectory() && !ignoredNames.match(file)) {
File subFiles[] = file.listFiles();
if (subFiles == null) {
logger.warning("Failed to get sub directories for '"
+ file.getAbsolutePath()
+ "', check access permissions.");
} else if (depth<=scanningDepth) {
addRepositories(subFiles, repos, ignoredNames, depth+1);
}
}
} else {
try {
String path = file.getCanonicalPath();
repository.setDirectoryName(path);
if (RuntimeEnvironment.getConfig().isVerbose()) {
logger.log(Level.FINE, "Adding <{0}> repository: <{1}>",
new Object[]{repository.getClass().getName(), path});
}
repos.add(new RepositoryInfo(repository));
// @TODO: Search only for one type of repository - the one found here
if (recursiveSearch && repository.supportsSubRepositories()) {
File subFiles[] = file.listFiles();
if (subFiles == null) {
logger.warning("Failed to get sub directories for '"
+ file.getAbsolutePath()
+ "', check access permissions.");
} else if (depth<=scanningDepth) {
// Search only one level down - if not: too much
// stat'ing for huge Mercurial repositories
addRepositories(subFiles, repos, ignoredNames,
false, depth+1);
}
}
} catch (IOException exp) {
logger.warning("Failed to get canonical path for '"
+ file.getAbsolutePath()
+ "'. Repository will be ignored: " + exp.getMessage());
logger.log(Level.FINE, "addRepositories", exp);
}
}
}
}
/**
* Search through the all of the directories and add all of the source
* repositories found.
*
* @param dir the root directory to start the search in.
*/
public void addRepositories(String dir) {
List<RepositoryInfo> repos = new ArrayList<RepositoryInfo>();
addRepositories(new File[] { new File(dir) }, repos,
RuntimeEnvironment.getConfig().getIgnoredNames(), 0);
RuntimeEnvironment.getConfig().setRepositories(repos);
invalidateRepositories(repos);
}
/**
* Update the source the contents in the source repositories.
*/
public void updateRepositories() {
boolean verbose = RuntimeEnvironment.getConfig().isVerbose();
for (Map.Entry<String, Repository> entry : repositories.entrySet()) {
Repository repository = entry.getValue();
String path = entry.getKey();
String type = repository.getClass().getSimpleName();
if (repository.isWorking()) {
if (verbose) {
logger.log(Level.INFO, "Update {0} repository in ''{1}''",
new String[] { type, path });
}
try {
repository.update();
} catch (UnsupportedOperationException e) {
logger.warning("Skipping update of " + type + " repository"
+ " in '" + path + "': Not implemented");
} catch (Exception e) {
logger.warning("An error occured while updating '"
+ path + "' (" + type + "): " + e.getMessage());
logger.log(Level.FINE, "updateRepositories", e);
}
} else {
logger.warning("Skipping update of " + type + " repository in '"
+ path + "': Missing SCM dependencies?");
}
}
}
/**
* Update the source the contents in the source repositories.
* @param paths A list of files/directories to update
*/
public void updateRepositories(Collection<String> paths) {
boolean verbose = RuntimeEnvironment.getConfig().isVerbose();
List<Repository> repos = getReposFromString(paths);
for (Repository repository : repos) {
String type = repository.getClass().getSimpleName();
if (repository.isWorking()) {
if (verbose) {
logger.log(Level.INFO, "Update {0} repository in ''{1}''",
new String[] { type, repository.getDirectoryName()});
}
try {
repository.update();
} catch (UnsupportedOperationException e) {
logger.warning("Skipping update of " + type + " repository"
+ " in '" + repository.getDirectoryName()
+ "': Not implemented");
} catch (Exception e) {
logger.warning("An error occured while updating '"
+ repository.getDirectoryName() + "' (" + type + "): "
+ e.getMessage());
logger.log(Level.FINE, "updateRepositories", e);
}
} else {
logger.warning("Skipping update of " + type + " repository in '"
+ repository.getDirectoryName() + "': Missing SCM dependencies?");
}
}
}
private void createCache(Repository repository, String sinceRevision) {
if (!useCache()) {
return;
}
String path = repository.getDirectoryName();
String type = repository.getClass().getSimpleName();
if (repository.isWorking()) {
boolean verbose = RuntimeEnvironment.getConfig().isVerbose();
long start = System.currentTimeMillis();
if (verbose) {
logger.log(Level.INFO, "Create historycache for ''{0}'' ({1})",
new String[]{path, type});
}
try {
repository.createCache(historyCache, sinceRevision);
} catch (Exception e) {
logger.warning("An error occured while creating cache for '"
+ path + "' (" + type + "): " + e.getMessage());
logger.log(Level.FINE, "createCache", e);
}
if (verbose) {
long stop = System.currentTimeMillis();
logger.log(Level.INFO, "Creating historycache for ''{0}'' took {1}ms",
new String[]{path, String.valueOf(stop - start)});
}
} else {
logger.warning("Skipping creation of historycache of "
+ type + " repository in '" + path + "': Missing SCM dependencies?");
}
}
/**
* Create the history cache for the given repositories, if caching is
* enabled and the repository in question supports history for directories.
* @param allRepos repositories in question. If {@code null} all registered
* repos are used instead.
*/
public void createCache(Collection<String> allRepos) {
if (!useCache()) {
return;
}
Collection<Repository> repositories = allRepos == null
? this.repositories.values()
: getReposFromString(allRepos);
int num = Runtime.getRuntime().availableProcessors() * 2;
String total = System.getProperty(Configuration.PROPERTY_KEY_PREFIX
+ "history.NumCacheThreads");
if (total != null) {
try {
num = Integer.parseInt(total, 10);
} catch (Throwable t) {
logger.warning("Failed to parse the number of cache threads to use for cache creation: "
+ t.getMessage());
logger.log(Level.FINE, "createCacheReal", t);
}
}
ExecutorService executor = Executors.newFixedThreadPool(num);
for (final Repository repos : repositories) {
final String latestRev;
try {
latestRev = historyCache.getLatestCachedRevision(repos);
} catch (HistoryException he) {
logger.warning("Failed to retrieve latest cached revision for '"
+ repos.getDirectoryName() + "': " + he.getMessage());
logger.log(Level.FINE, "createCacheReal", he);
continue;
}
executor.submit(new Runnable() {
@SuppressWarnings("synthetic-access")
@Override
public void run() {
createCache(repos, latestRev);
}
});
}
executor.shutdown();
while (!executor.isTerminated()) {
try {
// Wait forever
executor.awaitTermination(999, TimeUnit.DAYS);
} catch (InterruptedException exp) {
logger.warning("Received interrupt while waiting for executor to finish: "
+ exp.getMessage());
logger.log(Level.FINE, "createCacheReal", exp);
}
}
// The cache has been populated. Now, optimize how it is stored on
// disk to enhance performance and save space.
try {
historyCache.optimize();
} catch (HistoryException he) {
logger.warning("Failed optimizing the history cache database: "
+ he.getMessage());
logger.log(Level.FINE, "createCacheReal", he);
}
}
/**
* Remove the history cache for the given repositories.
* @param repositories repositories in question.
* @throws HistoryException
*/
public void removeCache(Collection<String> repositories) throws HistoryException {
List<Repository> repos = getReposFromString(repositories);
HistoryCache cache = historyCache;
if (cache == null) {
if (RuntimeEnvironment.getConfig().isHistoryCacheInDB()) {
cache = new JDBCHistoryCache();
cache.initialize();
} else {
cache = new FileHistoryCache();
}
}
for (Repository r : repos) {
try {
cache.clear(r);
logger.info("History cache for '" + r.getDirectoryName() + "' cleared.");
} catch (HistoryException e) {
logger.warning("Clearing history cache for repository '" +
r.getDirectoryName() + "' failed: " + e.getLocalizedMessage());
}
}
invalidateRepositories(repos);
}
/**
* Create the history cache for all of the repositories.
*/
public void createCache() {
createCache(null);
}
private List<Repository> getReposFromString(Collection<String> repositories) {
ArrayList<Repository> repos = new ArrayList<Repository>();
File root = RuntimeEnvironment.getConfig().getSourceRootFile();
for (String file : repositories) {
File f = new File(root, file);
Repository r = getRepository(f);
if (r == null) {
logger.warning("Could not locate a repository for '"
+ f.getAbsolutePath() + "'");
} else if (!repos.contains(r)){
repos.add(r);
}
}
return repos;
}
/**
* Check, whether the given path denotes a directory in the history cache.
* This really hits the DB, so caching is advised.
*
* @param path The <em>canonical</em> path to the file in question,
* e.g. sourceRoot + '/' + projectdir + '/' + subdir with '/'
* path separators, only! One trailing slash gets automatically removed
* if it exists.
* @return {@code null} if an error occured, {@code Boolean#TRUE} if there
* is a directory with the given name in the history cache,
* {@link Boolean#FALSE} otherwise.
*/
public Boolean isDirectory(String path) {
if (!useCache() || path == null) {
return null;
}
File dir = new File(path);
Repository repo = getRepository(dir);
try {
return Boolean.valueOf(historyCache.hasCacheForDirectory(dir, repo));
} catch (HistoryException e) {
logger.warning(e.getLocalizedMessage());
logger.log(Level.FINE, "isDirectory", e);
return null;
}
}
/**
* Ensure that we have a directory in the cache. If it's not there, fetch
* its history and populate the cache. If it's already there, and the
* cache is able to tell how recent it is, attempt to update it to the
* most recent revision.
*
* @param file the root path to test (canonical path incl. source root).
* @throws HistoryException if an error occurs while accessing the
* history cache
*/
public void ensureHistoryCacheExists(File file) throws HistoryException {
if (!useCache()) {
return;
}
Repository repository = getRepository(file);
if (repository == null) {
// no repository -> no history :(
return;
}
String sinceRevision = null;
if (historyCache.hasCacheForDirectory(file, repository)) {
sinceRevision = historyCache.getLatestCachedRevision(repository);
if (sinceRevision == null) {
// Cache already exists, but we don't know how recent it is,
// so don't do anything.
return;
}
}
// Create cache from the beginning if it doesn't exist, or update it
// incrementally otherwise.
createCache(getRepository(file), sinceRevision);
}
/**
* Get the with this instance registered repository for the given file.
* @param path canonical path incl. source root to the source file in
* question.
* @return {@code null} if unknown, the related repository otherwise.
*/
protected Repository getRepository(File path) {
Map<String, Repository> repos = repositories;
File file = path;
try {
file = path.getCanonicalFile();
} catch (IOException e) {
logger.warning("Failed to get canonical path for '" + path + "': "
+ e.getLocalizedMessage());
return null;
}
while (file != null) {
Repository r = repos.get(file.getAbsolutePath());
if (r != null) {
return r;
}
file = file.getParentFile();
}
return null;
}
/**
* Invalidate the current list of known repositories!
*
* @param repos The new repositories
*/
public void invalidateRepositories(Collection<? extends RepositoryInfo> repos)
{
if (repos == null || repos.isEmpty()) {
repositories.clear();
} else {
Map<String, Repository> nrep =
new HashMap<String, Repository>(repos.size());
for (RepositoryInfo i : repos) {
try {
Repository r = RepositoryFactory.getRepository(i);
if (r == null) {
logger.warning("Failed to instanciate internal repository data for "
+ i.getType() + " in '" + i.getDirectoryName() + "'");
} else {
nrep.put(r.getDirectoryName(), r);
}
} catch (InstantiationException ex) {
logger.warning("Could not create " + i.getType()
+ " for '" + i.getDirectoryName()
+ "', could not instantiate the repository: "
+ ex.getMessage());
logger.log(Level.FINE, "invalidateRepositories", ex);
} catch (IllegalAccessException iae) {
logger.warning("Could not create " + i.getType()
+ " for '" + i.getDirectoryName()
+ "', missing access rights: " + iae.getMessage());
logger.log(Level.FINE, "invalidateRepositories", iae);
}
}
repositories = nrep;
}
}
}