opengrok/history/MercurialRepository.java

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * See LICENSE.txt included in this distribution for the specific
 * language governing permissions and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at LICENSE.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2012, Oracle and/or its affiliates. All rights reserved.
 */
package org.opensolaris.opengrok.history;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.opensolaris.opengrok.configuration.Configuration;
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
import org.opensolaris.opengrok.util.Executor;
import org.opensolaris.opengrok.util.IOUtils;
import org.opensolaris.opengrok.util.StringUtils;
import org.opensolaris.opengrok.web.Util;

/**
 * Access to a Mercurial repository.
 */
public class MercurialRepository extends Repository {
    private static final Logger logger =
        Logger.getLogger(MercurialRepository.class.getName());
    private static final long serialVersionUID = 1L;

    /** The property name used to obtain the client command for thisrepository. */
    public static final String CMD_PROPERTY_KEY =
        Configuration.PROPERTY_KEY_PREFIX + "history.Mercurial";
    /** The command to use to access the repository if none was given explicitly */
    public static final String CMD_FALLBACK = "hg";

    /** The boolean property and environment variable name to indicate
     * whether forest-extension in Mercurial adds repositories inside the
     * repositories. */
    public static final String NOFOREST_PROPERTY_KEY =
        Configuration.PROPERTY_KEY_PREFIX + "history.mercurial.disableForest";

    /** Template for formatting hg log output for files. */
    private static final String[] TEMPLATE = {
        "changeset: {rev}:{node|short}\\n"
        + "{branches}{tags}{parents}\\n"
        + "user: {author}\\ndate: {date|rfc3339date}\\n"
        + "description: {desc|strip|obfuscate}\\n",

        "changeset: {rev}:{node|short}:{node}\\n"
        + "{branches}{tags}{parents}\\n"
        + "user: {author}\\ndate: {date|rfc3339date}\\n"
        + "description: {desc|strip|obfuscate}\\n"
    };

    /** Template for formatting hg log output for directories. */
    private static final String[] DIR_TEMPLATE = {
        TEMPLATE[0] + "files:\\n{files}\\n",
        TEMPLATE[1] + "files:\\n{files}\\n",
    };
    private static File[] STYLEFILE = new File[2];
    private static String DATE_PATTERN;

    // 0 .. normal repo, 1 .. bridged repo, i.e "cloned" from a different type
    private int style;
    private String svnUUID = null;

    private Map<String, String> oldRevMap;
    private long lastModOldRev;

    /**
     * Create a new instance of type {@code Mercurial}.
     */
    public MercurialRepository() {
        type = "Mercurial";
        if (DATE_PATTERN == null) {
            if (RuntimeEnvironment.isOldJVM()) {
                TEMPLATE[0] = TEMPLATE[0].replace("rfc3339date", "isodate");
                TEMPLATE[1] = TEMPLATE[1].replace("rfc3339date", "isodate");
                DIR_TEMPLATE[0] = DIR_TEMPLATE[0].replace("rfc3339date", "isodate");
                DIR_TEMPLATE[1] = DIR_TEMPLATE[1].replace("rfc3339date", "isodate");
                DATE_PATTERN = "yyyy-MM-dd hh:mm ZZZZ";
            } else {
                DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ssXXX";
            }
        }
        datePattern = DATE_PATTERN;
    }

    private static void checkStyleFile(int style) throws IOException {
        if (STYLEFILE[style] == null || !STYLEFILE[style].canRead()) {
            FileWriter out = null;
            String EOL = System.getProperty("line.separator");
            try {
                File f = File.createTempFile("hgstyle" + style, ".tmp", null);
                f.deleteOnExit();
                out = new FileWriter(f, false);
                out.write("changeset = \"");
                out.write(DIR_TEMPLATE[style]);
                out.write('"');
                out.write(EOL);
                // actually mercurial needs to provide a way to escape certain
                // chars in filenames, to get this 100% correct
                out.write("file = \"{file}\\n\"");
                out.write(EOL);
                out.flush();
                STYLEFILE[style] = f.getAbsoluteFile();
            } finally {
                IOUtils.close(out);
            }
        }
    }

    // package scope: MercurialHistoryParser.java is the only consumer
    /**
     * Get the mapping of this repository based revisions to the revisions of
     * the non-hybrid repository. Key is the full hg revision aka
     * 'node', the corresponding value is the revision string in the non-hybrid
     * repository.
     * @return an empty map, if this is a native mercurial repository or is not
     *  a hybrid (like hgsubversion), the fully populated map otherwise.
     */
    Map<String,String> getSrcRevisionMap() {
        if (oldRevMap != null && style != 1) {
            return oldRevMap;
        }
        File f = new File(directoryName, ".hg" + File.separator + "svn"
            + File.separator + "rev_map");
        if (! f.exists()) {
            oldRevMap = Collections.emptyMap();
            return oldRevMap;
        }
        if (oldRevMap != null) {
            long t = f.lastModified();
            if (t <= lastModOldRev) {
                return oldRevMap;
            }
            lastModOldRev = t;
        }
        Map<String, String> m = new HashMap<>();
        try (BufferedReader br = new BufferedReader(new FileReader(f))) {
            String line;
            while ((line = br.readLine()) != null) {
                int idx = line.indexOf(' ');
                if (idx > 0) {
                    m.put(line.substring(idx+1).trim(), line.substring(0, idx));
                }
            }
        } catch (IOException e) {
            logger.warning(e.getLocalizedMessage());
            if (logger.isLoggable(Level.FINE)) {
                logger.log(Level.FINE, "getSrcRevisionMap()", e);
            }
        }
        oldRevMap = m;
        return oldRevMap;
    }

    /**
     * Get the UUID assigned by hgsubversion to the repository.
     * @return {@code null} if this is not a subversion hybrid repository,
     *  the hgsubversion UUID otherwise.
     */
    public String getSvnUUID() {
        if (svnUUID == null) {
            File f = new File(directoryName, ".hg" + File.separator + "svn"
                + File.separator + "uuid");
            if (f.exists()) {
                try (BufferedReader br = new BufferedReader(new FileReader(f))){
                    String s = br.readLine();
                    svnUUID = s.trim();
                } catch (IOException e) {
                    logger.warning(e.getLocalizedMessage());
                    if (logger.isLoggable(Level.FINE)) {
                        logger.log(Level.FINE, "getSvnUUID()", e);
                    }
                }
            }
            if (svnUUID == null) {
                svnUUID = "";
                style = 0;
            } else {
                style = 1;
            }
        }
        return svnUUID.isEmpty() ? null : svnUUID;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void setDirectoryName(String directoryName) {
        if (this.directoryName != null
            && !StringUtils.isSame(this.directoryName, directoryName))
        {
            throw new IllegalArgumentException("Software bug: directory is "
                + "already set. (" + this.directoryName + " != "
                + directoryName + ")");
        }
        super.setDirectoryName(directoryName);
        getSvnUUID();   // trigger style adjustments
    }

    /**
     * Get an executor to be used for retrieving the history log for the
     * named file. When indexing it gets usually called twice: 1st without any
     * -r option, 2nd with -r $currentRev:tip.
     *
     * @param file The file to retrieve history for (canonical path incl. source
     *      root).
     * @param changeset the oldest changeset to return from the executor,
     * or {@code null} if all changesets should be returned
     * @return An Executor ready to be started
     */
    Executor getHistoryLogExecutor(File file, String changeset)
         throws HistoryException, IOException
    {
        String abs = file.getCanonicalPath();
        String filename = "";
        if (!abs.startsWith(directoryName)) {
            throw new HistoryException("File '" + file
                + "' doesn't belong to me (" + directoryName + ")");
        }
        if (abs.length() > directoryName.length()) {
            filename = abs.substring(directoryName.length() + 1);
        }

        List<String> cmd = new ArrayList<String>();
        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
        cmd.add(this.cmd);
        cmd.add("log");
        if ( !file.isDirectory() ) { cmd.add("-f"); }

        if (changeset != null) {
            cmd.add("-r");
            String[] parts = changeset.split(":");
            if (parts.length == 2 || (parts.length == 3 && style == 1)) {
                cmd.add("tip:" + parts[0]);
            } else {
                throw new HistoryException("Don't know how to parse changeset "
                    + changeset + " (" + file + " " + parts.length + " "
                    + getSvnUUID() + " " + directoryName + ")");
            }
        }

        if (file.isDirectory()) {
            checkStyleFile(style);
            cmd.add("--style");
            cmd.add(STYLEFILE[style].getPath());
        } else {
            cmd.add("--template");
            cmd.add(TEMPLATE[style]);
        }
        cmd.add(filename);

        return new Executor(cmd, new File(directoryName));
    }

    /**
     * {@inheritDoc}
     */
    @SuppressWarnings("resource")
    @Override
    public InputStream getHistoryGet(String parent, String basename, String rev)
    {
        InputStream ret = null;

        File directory = new File(directoryName);

        Process process = null;
        InputStream in = null;
        String revision = rev;

        if (rev.indexOf(':') != -1) {
            revision = rev.substring(0, rev.indexOf(':'));
        }
        try {
            String filename =  (new File(parent, basename)).getCanonicalPath();
            if (!filename.startsWith(directoryName)) {
                return null;
            }
            filename = filename.substring(directoryName.length() + 1);
            ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
            String argv[] = {cmd, "cat", "-r", revision, filename};
            process = Runtime.getRuntime().exec(argv, null, directory);

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            byte[] buffer = new byte[32 * 1024];
            in = process.getInputStream();
            int len;

            while ((len = in.read(buffer)) != -1) {
                if (len > 0) {
                    out.write(buffer, 0, len);
                }
            }

            ret = new ByteArrayInputStream(out.toByteArray());
        } catch (Exception exp) {
            logger.warning("Failed to get history: " + exp.getMessage());
        } finally {
            IOUtils.close(in);
            // Clean up zombie-processes...
            if (process != null) {
                try {
                    process.exitValue();
                } catch (IllegalThreadStateException exp) {
                    // the process is still running??? just kill it..
                    process.destroy();
                }
            }
        }

        return ret;
    }

    /** Pattern used to extract author/revision from hg annotate. */
    private static final Pattern ANNOTATION_PATTERN =
        Pattern.compile("^\\s*(\\d+):");

    /**
     * {@inheritDoc}
     */
    @Override
    public Annotation annotate(File file, String revision) throws IOException {
        ArrayList<String> argv = new ArrayList<String>();
        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
        argv.add(cmd);
        argv.add("annotate");
        argv.add("-n");
        if (revision != null) {
            argv.add("-r");
            if (revision.indexOf(':') == -1) {
                argv.add(revision);
            } else {
                argv.add(revision.substring(0, revision.indexOf(':')));
            }
        }
        argv.add(file.getName());
        ProcessBuilder pb = new ProcessBuilder(argv);
        pb.directory(file.getParentFile());
        Process process = null;
        BufferedReader in = null;
        Annotation ret = null;
        HashMap<String,HistoryEntry> revs = new HashMap<String,HistoryEntry>();

        // Construct hash map for history entries from history cache. This is
        // needed later to get user string for particular revision.
        try {
            History hist = HistoryGuru.getInstance().getHistory(file, false);
            for (HistoryEntry e : hist.getHistoryEntries()) {
                // Chop out the colon and all hexadecimal what follows.
                // This is because the whole changeset identification is
                // stored in history index while annotate only needs the
                // revision identifier.
                revs.put(e.getRevision().replaceFirst(":[a-f0-9]+", ""), e);
            }
        } catch (HistoryException he) {
            logger.warning("Cannot get history for file '" + file + "'");
            return null;
        }

        try {
            process = pb.start();
            in = new BufferedReader(new InputStreamReader(process.getInputStream()));
            ret = new Annotation(file.getName());
            String line;
            int lineno = 0;
            Matcher matcher = ANNOTATION_PATTERN.matcher("");
            HashMap<String, String> authorMap = new HashMap<String, String>();
            String uuid = getSvnUUID();
            if (uuid != null) {
                uuid = "@" + uuid;
            }
            while ((line = in.readLine()) != null) {
                ++lineno;
                matcher.reset(line);
                String lastRev = "";
                String author = "N/A";
                if (matcher.find()) {
                    String rev = matcher.group(1);
                    if (!lastRev.equals(rev)) {
                        // Use the history index hash map to get the author.
                        HistoryEntry he = revs.get(rev);
                        if (he != null) {
                             author = he.getAuthor();
                             // no different strings for the same author
                             String tmp = authorMap.get(author);
                             if (tmp == null) {
                                 // hgsubversioned entries have no e-mail
                                 tmp = (uuid != null)
                                     ? author
                                     : Util.getEmail(author);
                                 authorMap.put(author, tmp);
                             }
                             author = tmp;
                        }
                        lastRev = rev;
                    }
                    ret.addLine(rev, author);
                } else {
                    logger.warning("Did not find annotation in line "
                        + lineno + " [" + line + "]");
                }
            }
        } finally {
            IOUtils.close(in);
            if (process != null) {
                try {
                    process.exitValue();
                } catch (IllegalThreadStateException e) {
                    // the process is still running??? just kill it..
                    process.destroy();
                }
            }
        }
        return ret;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean fileHasAnnotation(File file) {
        return true;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void update() throws IOException {
        File directory = new File(directoryName);

        List<String> cmd = new ArrayList<String>();
        ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
        cmd.add(this.cmd);
        cmd.add("showconfig");
        Executor executor = new Executor(cmd, directory);
        if (executor.exec() != 0) {
            throw new IOException(executor.getErrorString());
        }

        if (executor.getOutputString().indexOf("paths.default=") != -1) {
            cmd.clear();
            cmd.add(this.cmd);
            cmd.add("pull");
            cmd.add("-u");
            executor = new Executor(cmd, directory);
            if (executor.exec() != 0) {
                throw new IOException(executor.getErrorString());
            }
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean fileHasHistory(File file) {
        // Todo: is there a cheap test for whether mercurial has history
        // available for a file?
        // Otherwise, this is harmless, since mercurial's commands will just
        // print nothing if there is no history.
        return true;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean isRepositoryFor(File file) {
        if (file.isDirectory()) {
            File f = new File(file, ".hg");
            return f.exists() && f.isDirectory();
        }
        return false;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean supportsSubRepositories() {
        String val = System.getenv(NOFOREST_PROPERTY_KEY);
        return !(val == null
            ? Boolean.getBoolean(NOFOREST_PROPERTY_KEY)
            : Boolean.parseBoolean(val));
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean isWorking() {
        if (working == null) {
            ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
            working = checkCmd(cmd);
        }
        return working.booleanValue();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean hasHistoryForDirectories() {
        return true;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public History getHistory(File file) throws HistoryException {
        return getHistory(file, null);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public History getHistory(File file, String sinceRevision)
        throws HistoryException
    {
        try {
            if (file == null
                || !file.getCanonicalPath().startsWith(getDirectoryName()))
            {
                return new History();
            }
        } catch (IOException e) {
            return new History();
        }
        return new MercurialHistoryParser(this).parse(file, sinceRevision);
    }
}