opengrok/history/GitRepository.java

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * See LICENSE.txt included in this distribution for the specific
 * language governing permissions and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at LICENSE.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2012, Oracle and/or its affiliates. All rights reserved.
 */
package org.opensolaris.opengrok.history;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.opensolaris.opengrok.configuration.Configuration;
import org.opensolaris.opengrok.util.Executor;
import org.opensolaris.opengrok.util.IOUtils;
import org.opensolaris.opengrok.util.StringUtils;

/**
 * Access to a Git repository.
 */
public class GitRepository extends Repository {

    private static final Logger logger =
        Logger.getLogger(GitRepository.class.getName());
    private static final long serialVersionUID = 1L;
    /** The property name used to obtain the client command for this repository. */
    public static final String CMD_PROPERTY_KEY =
        Configuration.PROPERTY_KEY_PREFIX + "history.git";
    /** The command to use to access the repository if none was given explicitly */
    public static final String CMD_FALLBACK = "git";

    /** git blame command */
    private static final String BLAME = "blame";
    /** the UUID of the bridged SVN repository */
    private String svnUUID = null;
    /** the svnRev2gitHash map - since git automat. appends it to the log
     * message, we don't need it yet. So for now it is here just for
     * completeness. */
    private Map<String, String> oldRevMap;
    /** LM date of the corresponding file (cached map refresh indicator) */
    private long lastModOldRev;

    /**
     * Create a new instance of type {@code git}.
     */
    public GitRepository() {
        type = "git";
        datePattern = "EEE MMM dd hh:mm:ss yyyy ZZZZ";
    }

    /**
     * Get path of the requested file given a commit hash.
     * Useful for tracking the path when file has changed its location.
     *
     * @param fileName name of the file to retrieve the path
     * @param revision commit hash to track the path of the file
     * @return full path of the file on success; null string on failure
     */
    private String getCorrectPath(String fileName, String revision)
        throws IOException
    {
        List<String> cmd = new ArrayList<String>();
        String path = "";

        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
        cmd.add(this.cmd);
        cmd.add(BLAME);
        cmd.add("-l");
        cmd.add("-C");
        cmd.add(fileName);
        File directory = new File(directoryName);
        Executor exec = new Executor(cmd, directory);

        int status = exec.exec();
        if (status != 0) {
            logger.warning("Failed to get blame list in resolving correct path");
            return path;
        }
        BufferedReader in = new BufferedReader(exec.getOutputReader());
        try {
            String pattern = "^\\W*" + revision + " (.+?) .*$";
            Pattern commitPattern = Pattern.compile(pattern);
            String line = "";
            Matcher matcher = commitPattern.matcher(line);
            while ((line = in.readLine()) != null) {
                matcher.reset(line);
                if (matcher.find()) {
                    path = matcher.group(1);
                    break;
                }
            }
        } finally {
            in.close();
        }

        return path;
    }

    /**
     * Get the mapping of this repository based revisions to the revisions of
     * the non-hybrid repository. Key is the full git revision aka 'hash', the
     * corresponding value is the revision string in the non-hybrid repository.
     * @return an empty map, if this is a native git repository or is not
     *      a hybrid (like git-svn), the fully populated map otherwise.
     */
    Map<String,String> getSrcRevisionMap() {
        if (oldRevMap != null) {
            return oldRevMap;
        }
        File f = new File(directoryName, ".git"
            + File.separator + "svn"
            + File.separator + "refs"
            + File.separator + "remotes"
            + File.separator + "git-svn"
            + File.separator + ".rev_map." + getSvnUUID());
        if (! f.exists()) {
            oldRevMap = Collections.emptyMap();
            return oldRevMap;
        }
        if (oldRevMap != null) {
            long t = f.lastModified();
            if (t <= lastModOldRev) {
                return oldRevMap;
            }
            lastModOldRev = t;
        }
        Map<String, String> m = new HashMap<>();
        try (DataInputStream dis =
            new DataInputStream(new BufferedInputStream(new FileInputStream(f))))
        {
            final char[] HEX = "0123456789abcdef".toCharArray();
            // NH40 => an uint32_t and a String of 40 hex characters (sha1)
            long rev;
            final byte[] hex_uuid = new byte[20];

            final char[] hash = new char[hex_uuid.length * 2];
            long count = f.length() / (4 + hex_uuid.length);
            int i, k;
            byte b;
            for (; count > 0; count--) {
                rev = 0xFFFFFFFFL & dis.readInt();
                dis.read(hex_uuid, 0, hex_uuid.length);
                for (i=0, k=0; i < hex_uuid.length; i++) {
                    b = hex_uuid[i];
                    hash[k++] = HEX[(b >> 4) & 0xF];
                    hash[k++] = HEX[(b & 0xF)];
                }
                m.put(new String(hash), Long.toString(rev, 10));
            }
            // sometimes the last record is a placeholder - the hash
            // is made of zeros, only.
            k = 0;
            for (i=0; i < hex_uuid.length; i++) {
                if (hex_uuid[i] != 0) {
                    k++;
                    break;
                }
            }
            if (k == 0) {
                m.remove(new String(hash));
            }
        } catch (IOException e) {
            logger.warning(e.getLocalizedMessage());
            if (logger.isLoggable(Level.FINE)) {
                logger.log(Level.FINE, "getSrcRevisionMap()", e);
            }
        }
        oldRevMap = m;
        return oldRevMap;
    }

    /**
     * Get the UUID assigned by git-svn to the repository.
     * @return {@code null} if this is not a subversion hybrid repository,
     * the git-svn UUID otherwise. */
    public String getSvnUUID() {
        if (svnUUID == null) {
            File f = new File(directoryName, ".git" + File.separator + "svn"
                + File.separator + ".metadata");
            if (f.exists()) {
                try (BufferedReader br = new BufferedReader(new FileReader(f))){
                    String s;
                    boolean svn_rem = false;
                    while ((s = br.readLine()) != null) {
                        if (! svn_rem) {
                            if (s.trim().equals("[svn-remote \"svn\"]")) {
                                svn_rem = true;
                            }
                            continue;
                        }
                        s = s.trim();
                        if (s.length() == 43 && s.startsWith("uuid = ")) {
                            svnUUID = s.substring(7);
                            break;
                        }
                    }
                } catch (IOException e) {
                    logger.warning(e.getLocalizedMessage());
                    if (logger.isLoggable(Level.FINE)) {
                        logger.log(Level.FINE, "getSvnUUID()", e);
                    }
                }
            }
            if (svnUUID == null) {
                svnUUID = "";
            }
        }
        return svnUUID.isEmpty() ? null : svnUUID;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void setDirectoryName(String directoryName) {
        if (!StringUtils.isSame(this.directoryName, directoryName)) {
            // should not happen twice, but if, make sure we flush the cache
            svnUUID = null;
            oldRevMap = null;
        }
        super.setDirectoryName(directoryName);
    }

    /**
     * Get an executor to be used for retrieving the history log for the
     * named file.
     *
     * @param file The file to retrieve history for (canonical path incl. source
     *  root).
     * @return An Executor ready to be started
     */
    Executor getHistoryLogExecutor(final File file, String sinceRevision)
        throws IOException
    {
        String abs = file.getCanonicalPath();
        String filename = "";
        if (abs.length() > directoryName.length()) {
            filename = abs.substring(directoryName.length() + 1);
        }

        List<String> cmd = new ArrayList<String>();
        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
        cmd.add(this.cmd);
        cmd.add("log");
        cmd.add("--name-only");
        cmd.add("--pretty=fuller");

        if (sinceRevision != null) {
            cmd.add(sinceRevision + "..");
        }

        if (filename.length() > 0) {
            cmd.add(filename);
        }
        return new Executor(cmd, new File(getDirectoryName()));
    }

    /**
     * Create a {@code Reader} that reads an {@code InputStream} using the
     * correct character encoding.
     *
     * @param input a stream with the output from a log or blame command
     * @return a reader that reads the input
     * @throws IOException if the reader cannot be created
     */
    @SuppressWarnings("static-method")
    Reader newLogReader(InputStream input) throws IOException {
        // Bug #17731: Git always encodes the log output using UTF-8 (unless
        // overridden by i18n.logoutputencoding, but let's assume that hasn't
        // been done for now). Create a reader that uses UTF-8 instead of the
        // platform's default encoding.
        return new InputStreamReader(input, "UTF-8");
    }

    /**
     * {@inheritDoc}
     */
    @SuppressWarnings("resource")
    @Override
    public InputStream getHistoryGet(String parent, String basename, String rev)
    {
        InputStream ret = null;

        File directory = new File(directoryName);

        ByteArrayOutputStream output = new ByteArrayOutputStream();
        byte[] buffer = new byte[8192];

        Process process = null;
        InputStream in = null;
        try {
            String filename = (new File(parent, basename)).getCanonicalPath()
                .substring(directoryName.length() + 1);
            ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
            String argv[] = {cmd, "show", rev + ":" + filename};
            process = Runtime.getRuntime().exec(argv, null, directory);

            in = process.getInputStream();
            int len;
            boolean error = true;

            while ((len = in.read(buffer)) != -1) {
                error = false;
                if (len > 0) {
                    output.write(buffer, 0, len);
                }
            }
            if (error) {
                process.destroy();
                String path = getCorrectPath(filename, rev);
                argv[2] = rev + ":" + path;
                process = Runtime.getRuntime().exec(argv, null, directory);
                in = process.getInputStream();
                while ((len = in.read(buffer)) != -1) {
                    if (len > 0) {
                        output.write(buffer, 0, len);
                    }
                }
            }

            ret = new ByteArrayInputStream(output.toByteArray());
        } catch (Exception exp) {
            logger.warning("Failed to get history: " + exp.getMessage());
            logger.log(Level.FINE, "getHistoryGet", exp);
        } finally {
            // Clean up zombie-processes...
            if (process != null) {
                try {
                    process.exitValue();
                } catch (IllegalThreadStateException exp) {
                    // the process is still running??? just kill it..
                    process.destroy();
                }
            }
            IOUtils.close(in);
        }

        return ret;
    }
    /** Pattern used to extract author/revision from git blame. */
    private static final Pattern BLAME_PATTERN =
        Pattern.compile("^\\W*(\\w+).+?\\((\\D+).*$");

    /**
     * {@inheritDoc}
     */
    @Override
    public Annotation annotate(File file, String revision) throws IOException {
        List<String> cmd = new ArrayList<String>();
        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
        cmd.add(this.cmd);
        cmd.add(BLAME);
        cmd.add("-l");
        if (revision != null) {
            cmd.add(revision);
        }
        cmd.add(file.getName());

        Executor exec = new Executor(cmd, file.getParentFile());
        int status = exec.exec();

        // File might have changed its location
        if (status != 0) {
            cmd.clear();
            ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
            cmd.add(this.cmd);
            cmd.add(BLAME);
            cmd.add("-l");
            cmd.add("-C");
            cmd.add(file.getName());
            exec = new Executor(cmd, file.getParentFile());
            status = exec.exec();
            if (status != 0) {
                logger.warning("Failed to get blame list");
            }
            BufferedReader in = new BufferedReader(exec.getOutputReader());
            try {
                String pattern = "^\\W*" + revision + " (.+?) .*$";
                Pattern commitPattern = Pattern.compile(pattern);
                String line = "";
                Matcher matcher = commitPattern.matcher(line);
                while ((line = in.readLine()) != null) {
                    matcher.reset(line);
                    if (matcher.find()) {
                        String filepath = matcher.group(1);
                        cmd.clear();
                        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
                        cmd.add(this.cmd);
                        cmd.add(BLAME);
                        cmd.add("-l");
                        if (revision != null) {
                            cmd.add(revision);
                        }
                        cmd.add("--");
                        cmd.add(filepath);
                        File directory = new File(directoryName);
                        exec = new Executor(cmd, directory);
                        status = exec.exec();
                        if (status != 0) {
                            logger.warning("Failed to get blame details for modified file path");
                        }
                        break;
                    }
                }
            } finally {
                in.close();
            }
        }

        if (status != 0) {
            logger.warning("Failed to get annotations for '"
                + file.getAbsolutePath() + "' - Exit code " + status);
        }

        return parseAnnotation(newLogReader(exec.getOutputStream()), file.getName());
    }

    /**
     * Parse the given input for annotation infos.
     * @param input     data to parse
     * @param fileName  name of the file associated with the given input (used
     *  for reporting, only)
     * @return a annotion which may or may not contain all reqiuired information.
     * @throws IOException
     */
    private static Annotation parseAnnotation(Reader input, String fileName)
        throws IOException
    {
        BufferedReader in = new BufferedReader(input);
        Annotation ret = new Annotation(fileName);
        String line = "";
        int lineno = 0;
        Matcher matcher = BLAME_PATTERN.matcher(line);
        while ((line = in.readLine()) != null) {
            ++lineno;
            matcher.reset(line);
            if (matcher.find()) {
                String rev = matcher.group(1);
                String author = matcher.group(2).trim();
                ret.addLine(rev, author);
            } else {
                logger.log(Level.WARNING,
                    "Did not find annotation in line {0} [{1}] of {2}",
                    new Object[] { String.valueOf(lineno), line, fileName });
            }
        }
        return ret;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean fileHasAnnotation(File file) {
        return true;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void update() throws IOException {
        File directory = new File(getDirectoryName());
        List<String> cmd = new ArrayList<String>();
        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
        cmd.add(this.cmd);
        cmd.add("config");
        cmd.add("--list");

        Executor executor = new Executor(cmd, directory);
        if (executor.exec() != 0) {
            throw new IOException(executor.getErrorString());
        }

        if (executor.getOutputString().indexOf("remote.origin.url=") != -1) {
            cmd.clear();
            cmd.add(this.cmd);
            cmd.add("pull");
            cmd.add("-n");
            cmd.add("-q");
            if (executor.exec() != 0) {
                throw new IOException(executor.getErrorString());
            }
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean fileHasHistory(File file) {
        // Todo: is there a cheap test for whether Git has history
        // available for a file?
        // Otherwise, this is harmless, since Git's commands will just
        // print nothing if there is no history.
        return true;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean isRepositoryFor(File file) {
        if (file.isDirectory()) {
            File f = new File(file, ".git");
            return f.exists() && f.isDirectory();
        }
        return false;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean isWorking() {
        if (working == null) {
            ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
            working = checkCmd(cmd, "--help");
        }
        return working.booleanValue();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean hasHistoryForDirectories() {
        return true;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public History getHistory(File file) throws HistoryException {
        return getHistory(file, null);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public History getHistory(File file, String sinceRevision)
        throws HistoryException
    {
        return new GitHistoryParser().parse(file, this, sinceRevision);
    }
}