opengrok/history/JDBCHistoryCache.java

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * See LICENSE.txt included in this distribution for the specific
 * language governing permissions and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at LICENSE.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved.
 */
package org.opensolaris.opengrok.history;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLTransientException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
import org.opensolaris.opengrok.jdbc.ConnectionManager;
import org.opensolaris.opengrok.jdbc.ConnectionResource;
import org.opensolaris.opengrok.jdbc.InsertQuery;
import org.opensolaris.opengrok.jdbc.PreparedQuery;
import org.opensolaris.opengrok.logger.LoggerFactory;

class JDBCHistoryCache implements HistoryCache {

    private static final Logger LOGGER = LoggerFactory.getLogger(JDBCHistoryCache.class);

    private boolean historyIndexDone = false;

    /**
     * The schema in which the tables live.
     */
    private static final String SCHEMA = "OPENGROK";

    /**
     * The names of all the tables created by this class.
     */
    private static final String[] TABLES = {
        "REPOSITORIES", "FILES", "AUTHORS", "CHANGESETS", "FILECHANGES",
        "DIRECTORIES", "DIRCHANGES"
    };

    private static final Properties QUERIES = new Properties();

    /**
     * SQL queries used by this class.
     */
    static {
        Class<?> klazz = JDBCHistoryCache.class;
        try (InputStream in = klazz.getResourceAsStream(
                klazz.getSimpleName() + "_queries.properties")) {
            if (in != null) {
                QUERIES.load(in);
            }
        } catch (IOException ioe) {
            throw new ExceptionInInitializerError(ioe);
        }
    }

    private static final PreparedQuery GET_AUTHORS
            = new PreparedQuery(getQuery("getAuthors"));

    private static final InsertQuery ADD_AUTHOR
            = new InsertQuery(getQuery("addAuthor"));

    private static final PreparedQuery GET_DIRS
            = new PreparedQuery(getQuery("getDirectories"));

    private static final PreparedQuery GET_FILES
            = new PreparedQuery(getQuery("getFiles"));

    private static final InsertQuery INSERT_DIR
            = new InsertQuery(getQuery("addDirectory"));

    private static final InsertQuery INSERT_FILE
            = new InsertQuery(getQuery("addFile"));

    private static final PreparedQuery GET_LATEST_REVISION
            = new PreparedQuery(getQuery("getLatestCachedRevision"));

    private static final PreparedQuery GET_LAST_MODIFIED_TIMES
            = new PreparedQuery(getQuery("getLastModifiedTimes"));

    private static final PreparedQuery GET_FILEMOVES_COUNT
            = new PreparedQuery(getQuery("getFilemovesCount"));

    /**
     * The number of times to retry an operation that failed in a way that
     * indicates that it may succeed if it's tried again.
     */
    private static final int MAX_RETRIES = 2;

    /**
     * The maximum number of characters in commit messages. Longer messages will
     * be truncated.
     */
    private static final int MAX_MESSAGE_LENGTH = 32672;

    private ConnectionManager connectionManager;

    private final String jdbcDriverClass;
    private final String jdbcConnectionURL;

    /**
     * The id to be used for the next row inserted into FILES.
     */
    private final AtomicInteger nextFileId = new AtomicInteger();

    /**
     * The id to be used for the next row inserted into DIRECTORIES.
     */
    private final AtomicInteger nextDirId = new AtomicInteger();

    /**
     * The id to be used for the next row inserted into CHANGESETS.
     */
    private final AtomicInteger nextChangesetId = new AtomicInteger();

    /**
     * The id to be used for the next row inserted into AUTHORS.
     */
    private final AtomicInteger nextAuthorId = new AtomicInteger();

    /**
     * Info string to return from {@link #getInfo()}.
     */
    private String info;

    /**
     * Create a new cache instance with the default JDBC driver and URL.
     */
    JDBCHistoryCache() {
        this(RuntimeEnvironment.getInstance().getDatabaseDriver(),
                RuntimeEnvironment.getInstance().getDatabaseUrl());
    }

    /**
     * Create a new cache instance with the specified JDBC driver and URL.
     *
     * @param jdbcDriverClass JDBC driver class to access the database backend
     * @param url the JDBC url to the database
     */
    JDBCHistoryCache(String jdbcDriverClass, String url) {
        this.jdbcDriverClass = jdbcDriverClass;
        this.jdbcConnectionURL = url;
    }

    /**
     * Check whether this cache implementation can store history for the given
     * repository. Only repositories that support retrieval of history for the
     * whole directory at once are supported.
     */
    @Override
    public boolean supportsRepository(Repository repository) {
        return repository.hasHistoryForDirectories();
    }

    /**
     * Handle an {@code SQLException}. If the exception indicates that the
     * operation may succeed if it's retried and the number of attempts hasn't
     * exceeded the limit defined by {@link #MAX_RETRIES}, ignore it and let the
     * caller retry the operation. Otherwise, re-throw the exception.
     *
     * @param sqle the exception to handle
     * @param attemptNo the attempt number, first attempt is 0
     * @throws SQLException if the operation shouldn't be retried
     */
    private static void handleSQLException(SQLException sqle, int attemptNo)
            throws SQLException {
        boolean isTransient = false;
        for (Throwable cause : sqle) {
            if (cause instanceof SQLTransientException) {
                isTransient = true;
                break;
            }
        }

        if (isTransient && attemptNo < MAX_RETRIES) {
            LOGGER.info("Transient database failure detected. Retrying.");
            LOGGER.log(Level.FINE, "Transient database failure details:", sqle);
        } else {
            throw sqle;
        }
    }

    /**
     * Get the SQL text for a name query.
     *
     * @param key name of the query
     * @return SQL text for the query
     */
    private static String getQuery(String key) {
        return QUERIES.getProperty(key);
    }

    private void initDB(Statement s) throws SQLException {
        // TODO Store a database version which is incremented on each
        // format change. When a version change is detected, drop the database
        // or, if possible, upgrade the database to the new format. For now,
        // check if the tables exist, and create them if necessary.

        DatabaseMetaData dmd = s.getConnection().getMetaData();

        if (!tableExists(dmd, SCHEMA, "REPOSITORIES")) {
            s.execute(getQuery("createTableRepositories"));
        }

        if (!tableExists(dmd, SCHEMA, "DIRECTORIES")) {
            s.execute(getQuery("createTableDirectories"));
        }

        // Databases created with 0.11 or earlier versions don't have a
        // PARENT column in the DIRECTORIES table. If the column is missing,
        // create it and populate it. Bug #3174.
        if (!columnExists(dmd, SCHEMA, "DIRECTORIES", "PARENT")) {
            s.execute(getQuery("alterTableDirectoriesParent"));
            s.execute(getQuery("alterTableDirectoriesParentPathConstraint"));
            fillDirectoriesParentColumn(s);
        }

        if (!tableExists(dmd, SCHEMA, "FILES")) {
            s.execute(getQuery("createTableFiles"));
        }

        if (!tableExists(dmd, SCHEMA, "AUTHORS")) {
            s.execute(getQuery("createTableAuthors"));
        }

        if (!tableExists(dmd, SCHEMA, "CHANGESETS")) {
            s.execute(getQuery("createTableChangesets"));
            // Create a composite index on the repository in ascending order
            // and the id in descending order. This index may allow faster
            // retrieval of history in reverse chronological order.
            s.execute(getQuery("createIndexChangesetsRepoIdDesc"));
        }

        if (!tableExists(dmd, SCHEMA, "DIRCHANGES")) {
            s.execute(getQuery("createTableDirchanges"));
        }

        if (!tableExists(dmd, SCHEMA, "FILECHANGES")) {
            s.execute(getQuery("createTableFilechanges"));
        }

        if (!tableExists(dmd, SCHEMA, "FILEMOVES")) {
            s.execute(getQuery("createTableFilemoves"));
        }

        // Derby has some performance problems with auto-generated identity
        // columns when multiple threads insert into the same table
        // concurrently. Therefore, we have our own light-weight id generators
        // that we initialize on start-up. Details can be found in Derby's
        // bug tracker: https://issues.apache.org/jira/browse/DERBY-4437
        initIdGenerator(s, "getMaxFileId", nextFileId);
        initIdGenerator(s, "getMaxDirId", nextDirId);
        initIdGenerator(s, "getMaxChangesetId", nextChangesetId);
        initIdGenerator(s, "getMaxAuthorId", nextAuthorId);

        StringBuilder infoBuilder = new StringBuilder();
        infoBuilder.append(getClass().getSimpleName()).append("\n");
        infoBuilder.append("Driver class: ").append(jdbcDriverClass).
                append("\n");
        infoBuilder.append("URL: ").append(jdbcConnectionURL).append("\n");
        infoBuilder.append("Database name: ").
                append(dmd.getDatabaseProductName()).append("\n");
        infoBuilder.append("Database version: ").
                append(dmd.getDatabaseProductVersion()).append("\n");
        info = infoBuilder.toString();
    }

    /**
     * Fill the PARENT column of the DIRECTORIES table with correct values. Used
     * when upgrading a database from an old format that doesn't have the PARENT
     * column.
     */
    private static void fillDirectoriesParentColumn(Statement s)
            throws SQLException {
        try (PreparedStatement update = s.getConnection().prepareStatement(
                getQuery("updateDirectoriesParent"));
                ResultSet rs = s.executeQuery(getQuery("getAllDirectories"))) {
            while (rs.next()) {
                update.setInt(1, rs.getInt("REPOSITORY"));
                update.setString(2, getParentPath(rs.getString("PATH")));
                update.setInt(3, rs.getInt("ID"));
                update.executeUpdate();
            }
        }
    }

    private static boolean tableExists(
            DatabaseMetaData dmd, String schema, String table)
            throws SQLException {
        try (ResultSet rs = dmd.getTables(
                null, schema, table, new String[]{"TABLE"})) {
            return rs.next();
        }
    }

    private static boolean columnExists(
            DatabaseMetaData dmd, String schema, String table, String column)
            throws SQLException {
        try (ResultSet rs = dmd.getColumns(null, schema, table, column)) {
            return rs.next();
        }
    }

    /**
     * Initialize the {@code AtomicInteger} object that holds the value of the
     * id to use for the next row in a certain table. If there are rows in the
     * table, take the maximum value and increment it by one. Otherwise, the
     * {@code AtomicInteger} will be left at its current value (presumably 0).
     *
     * @param s a statement object on which the max query is executed
     * @param stmtKey name of the query to execute in order to get max id
     * @param generator the {@code AtomicInteger} object to initialize
     */
    private static void initIdGenerator(
            Statement s, String stmtKey, AtomicInteger generator)
            throws SQLException {
        try (ResultSet rs = s.executeQuery(getQuery(stmtKey))) {
            if (rs.next()) {
                int val = rs.getInt(1);
                if (!rs.wasNull()) {
                    generator.set(val + 1);
                }
            }
        }
    }

    @Override
    public void initialize() throws HistoryException {
        try {
            connectionManager
                    = new ConnectionManager(jdbcDriverClass, jdbcConnectionURL);
            for (int i = 0;; i++) {
                final ConnectionResource conn
                        = connectionManager.getConnectionResource();
                try {
                    try (Statement stmt = conn.createStatement()) {
                        initDB(stmt);
                    }
                    conn.commit();
                    // Success! Break out of the loop.
                    return;
                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                } finally {
                    connectionManager.releaseConnection(conn);
                }
            }
        } catch (Exception e) {
            throw new HistoryException(e);
        }
    }

    private static final PreparedQuery IS_DIR_IN_CACHE
            = new PreparedQuery(getQuery("hasCacheForDirectory"));

    // We do check the return value from ResultSet.next(), but PMD doesn't
    // understand it, so suppress the warning.
    @SuppressWarnings("PMD.CheckResultSet")
    @Override
    public boolean hasCacheForDirectory(File file, Repository repository)
            throws HistoryException {
        assert file.isDirectory();
        try {
            for (int i = 0;; i++) {
                final ConnectionResource conn
                        = connectionManager.getConnectionResource();
                try {
                    PreparedStatement ps = conn.getStatement(IS_DIR_IN_CACHE);
                    ps.setString(1, toUnixPath(repository.getDirectoryName()));
                    ps.setString(2, getSourceRootRelativePath(file));
                    try (ResultSet rs = ps.executeQuery()) {
                        return rs.next();
                    }
                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                } finally {
                    connectionManager.releaseConnection(conn);
                }
            }
        } catch (SQLException sqle) {
            throw new HistoryException(sqle);
        }
    }

    /**
     * Get path name with all file separators replaced with '/'.
     */
    private static String toUnixPath(String path) {
        return path.replace(File.separatorChar, '/');
    }

    /**
     * Get path name with all file separators replaced with '/'.
     */
    private static String toUnixPath(File file) throws HistoryException {
        try {
            return toUnixPath(file.getCanonicalPath());
        } catch (IOException ioe) {
            throw new HistoryException(ioe);
        }
    }

    /**
     * Get the path of a file relative to the source root.
     *
     * @param file the file to get the path for
     * @return relative path for {@code file} with unix file separators
     */
    private static String getSourceRootRelativePath(File file)
            throws HistoryException {
        String filePath = toUnixPath(file);
        String rootPath = RuntimeEnvironment.getInstance().getSourceRootPath();
        return getRelativePath(filePath, rootPath);
    }

    /**
     * Get the path of a file relative to the specified root directory.
     *
     * @param filePath the canonical path of the file to get the relative path
     * for
     * @param rootPath the canonical path of the root directory
     * @return relative path with unix file separators
     */
    private static String getRelativePath(String filePath, String rootPath) {
        assert filePath.startsWith(rootPath);
        return filePath.substring(rootPath.length());
    }

    /**
     * Get the base name of a path (with unix file separators).
     *
     * @param fullPath the full path of the file with unix file separators
     * @return the base name of the file
     */
    private static String getBaseName(String fullPath) {
        int idx = fullPath.lastIndexOf('/');
        return (idx >= 0) ? fullPath.substring(idx + 1) : fullPath;
    }

    /**
     * Get the path to the parent of the specified file.
     *
     * @param fullPath the full path of the file with unix file separators
     * @return the full path of the file's parent
     */
    private static String getParentPath(String fullPath) {
        int idx = fullPath.lastIndexOf('/');
        return (idx >= 0) ? fullPath.substring(0, idx) : fullPath;
    }

    /**
     * Split a full (unix-style) path into an array of path elements.
     *
     * @param fullPath the full unix-style path name
     * @return an array with each separate element of the path
     * @throws IllegalArgumentException if fullPath doesn't start with '/'
     */
    private static String[] splitPath(String fullPath) {
        if (fullPath.isEmpty() || fullPath.charAt(0) != '/') {
            throw new IllegalArgumentException("Not a full path: " + fullPath);
        }
        return fullPath.substring(1).split("/");
    }

    /**
     * Reconstruct a path previously split by {@link #splitPath(String)}, or
     * possibly just a part of it (only the {@code num} first elements will be
     * used).
     *
     * @param pathElts the elements of the path
     * @param num the number of elements to use when reconstructing the path
     * @return a path name
     */
    private static String unsplitPath(String[] pathElts, int num) {
        StringBuilder out = new StringBuilder("");
        for (int i = 0; i < num; i++) {
            out.append("/").append(pathElts[i]);
        }
        return out.toString();
    }

    /**
     * Truncate a string to the given length.
     *
     * @param str the string to truncate
     * @param length the length of the string after truncation
     * @return the truncated string
     * @throws IllegalArgumentException if the string is not longer than the
     * specified length
     */
    private static String truncate(String str, int length) {
        if (str.length() < length) {
            throw new IllegalArgumentException();
        }
        String suffix = " (...)";
        return length < suffix.length()
                ? str.substring(0, length)
                : (str.substring(0, length - suffix.length()) + suffix);
    }

    /**
     * Statement that gets the history for the specified file and repository.
     * The result is ordered in reverse chronological order to match the
     * required ordering for {@link HistoryCache#get(File, Repository)}.
     */
    private static final PreparedQuery GET_FILE_HISTORY
            = new PreparedQuery(getQuery("getFileHistory"));
    private static final PreparedQuery GET_FILE_HISTORY_FOLDED
            = new PreparedQuery(getQuery("getFileHistoryFolded"));

    /**
     * Statement that gets the history for all files matching a pattern in the
     * given repository. The result is ordered in reverse chronological order to
     * match the required ordering for
     * {@link HistoryCache#get(File, Repository)}.
     */
    private static final PreparedQuery GET_DIR_HISTORY
            = new PreparedQuery(getQuery("getDirHistory"));

    /**
     * Statement that retrieves all the files touched by a given changeset.
     */
    private static final PreparedQuery GET_CS_FILES
            = new PreparedQuery(getQuery("getFilesInChangeset"));

    /**
     * Statement for getting ID of given revision
     */
    private static final PreparedQuery GET_REV_ID
            = new PreparedQuery(getQuery("getChangesetIdForRevision"));

    @Override
    public History get(File file, Repository repository, boolean withFiles)
            throws HistoryException {
        try {
            for (int i = 0;; i++) {
                try {
                    return getHistory(file, repository, withFiles);
                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                }
            }
        } catch (SQLException sqle) {
            throw new HistoryException(sqle);
        }
    }

    /**
     * Get the number of rows in the FILEMOVES table. This is used as a
     * workaround/optimization since JavaDB cannot currently handle the
     * GET_FILE_HISTORY very well.
     *
     * @return number of rows in the FILEMOVES table
     * @throws SQLException
     */
    private int getFilemovesCount() throws SQLException {
        final ConnectionResource conn;
        conn = connectionManager.getConnectionResource();

        try {
            final PreparedStatement cntPS = conn.getStatement(GET_FILEMOVES_COUNT);
            try (ResultSet rs = cntPS.executeQuery()) {
                if (rs.next()) {
                    return rs.getInt(1);
                }
            }
        } finally {
            connectionManager.releaseConnection(conn);
        }

        return -1;
    }

    /**
     * Helper for {@link #get(File, Repository)}.
     */
    private History getHistory(
            File file, Repository repository, boolean withFiles)
            throws HistoryException, SQLException {
        final String filePath = getSourceRootRelativePath(file);
        final String reposPath = toUnixPath(repository.getDirectoryName());
        final ArrayList<HistoryEntry> entries = new ArrayList<>();
        final ConnectionResource conn
                = connectionManager.getConnectionResource();
        RuntimeEnvironment env = RuntimeEnvironment.getInstance();

        try {
            final PreparedStatement ps;
            if (file.isDirectory()) {
                // Fetch history for all files under this directory.
                ps = conn.getStatement(GET_DIR_HISTORY);
                ps.setString(2, filePath);
            } else {
                // Fetch history for a single file only.
                ps = conn.getStatement(env.isHandleHistoryOfRenamedFiles()
                        && (getFilemovesCount() > 0)
                                ? GET_FILE_HISTORY : GET_FILE_HISTORY_FOLDED);
                ps.setString(2, getParentPath(filePath));
                ps.setString(3, getBaseName(filePath));
            }
            ps.setString(1, reposPath);

            final PreparedStatement filePS
                    = withFiles ? conn.getStatement(GET_CS_FILES) : null;

            try (ResultSet rs = ps.executeQuery()) {
                while (rs.next()) {
                    // Get the information about a changeset
                    String revision = rs.getString(1);
                    String author = rs.getString(2);
                    Timestamp time = rs.getTimestamp(3);
                    String message = rs.getString(4);
                    HistoryEntry entry = new HistoryEntry(
                            revision, time, author, null, message, true);
                    entries.add(entry);

                    // Fill the list of files touched by the changeset, if
                    // requested.
                    if (withFiles) {
                        int changeset = rs.getInt(5);
                        filePS.setInt(1, changeset);
                        try (ResultSet fileRS = filePS.executeQuery()) {
                            while (fileRS.next()) {
                                entry.addFile(fileRS.getString(1));
                            }
                        }
                    }
                }
            }
        } finally {
            connectionManager.releaseConnection(conn);
        }

        History history = new History();
        history.setHistoryEntries(entries);

        if (env.isTagsEnabled() && repository.hasFileBasedTags()) {
            repository.assignTagsInHistory(history);
        }

        return history;
    }

    private static final PreparedQuery GET_REPOSITORY
            = new PreparedQuery(getQuery("getRepository"));

    private static final InsertQuery INSERT_REPOSITORY
            = new InsertQuery(getQuery("addRepository"));

    /**
     * store history for repository. Note that after this method returns it is
     * not guaranteed that the data will be returned in full in get() method
     * since some of the threads can be still running.
     */
    @Override
    public void store(History history, Repository repository)
            throws HistoryException {
        try {
            final ConnectionResource conn
                    = connectionManager.getConnectionResource();
            try {
                storeHistory(conn, history, repository);
            } finally {
                connectionManager.releaseConnection(conn);
            }
        } catch (SQLException sqle) {
            throw new HistoryException(sqle);
        }
    }

    private static final InsertQuery ADD_CHANGESET
            = new InsertQuery(getQuery("addChangeset"));

    private static final PreparedQuery ADD_DIRCHANGE
            = new PreparedQuery(getQuery("addDirchange"));

    private static final PreparedQuery ADD_FILECHANGE
            = new PreparedQuery(getQuery("addFilechange"));

    private static final PreparedQuery ADD_FILEMOVE
            = new PreparedQuery(getQuery("addFilemove"));

    /**
     * Get ID value for revision string by querying the DB.
     *
     * @param revision
     * @return ID
     */
    private int getIdForRevision(String revision, int repoId) throws SQLException {
        final ConnectionResource conn
                = connectionManager.getConnectionResource();
        try {
            PreparedStatement ps = conn.getStatement(GET_REV_ID);
            ps.setString(1, revision);
            ps.setInt(2, repoId);
            ResultSet rs = ps.executeQuery();
            return rs.next() ? Integer.valueOf(rs.getString(1)) : -1;
        } catch (java.sql.SQLException e) {
            LOGGER.log(Level.WARNING,
                    "getIdForRevision exception{0}", e);
            return -1;
        } finally {
            connectionManager.releaseConnection(conn);
        }
    }

    private void storeHistory(final ConnectionResource conn, History history,
            final Repository repository) throws SQLException {

        Integer reposId = null;
        Map<String, Integer> authors = null;
        Map<String, Integer> filesNf = null;
        final Map<String, Integer> files;
        Map<String, Integer> directories = null;
        PreparedStatement addChangeset = null;
        PreparedStatement addDirchange = null;
        PreparedStatement addFilechange = null;
        RuntimeEnvironment env = RuntimeEnvironment.getInstance();

        // Return immediately when there is nothing to do.
        List<HistoryEntry> entries = history.getHistoryEntries();
        if (entries.isEmpty()) {
            return;
        }

        LOGGER.log(Level.FINE,
                "Storing history for repo {0}",
                new Object[]{repository.getDirectoryName()});

        for (int i = 0;; i++) {
            try {
                if (reposId == null) {
                    reposId = getRepositoryId(conn, repository);
                    conn.commit();
                }

                if (authors == null) {
                    authors = getAuthors(conn, history, reposId);
                    conn.commit();
                }

                if (directories == null || filesNf == null) {
                    Map<String, Integer> dirs = new HashMap<>();
                    Map<String, Integer> fls = new HashMap<>();
                    getFilesAndDirectories(conn, history, reposId, dirs, fls);
                    conn.commit();
                    directories = dirs;
                    filesNf = fls;
                }

                if (addChangeset == null) {
                    addChangeset = conn.getStatement(ADD_CHANGESET);
                }

                if (addDirchange == null) {
                    addDirchange = conn.getStatement(ADD_DIRCHANGE);
                }

                if (addFilechange == null) {
                    addFilechange = conn.getStatement(ADD_FILECHANGE);
                }

                // Success! Break out of the loop.
                break;

            } catch (SQLException sqle) {
                handleSQLException(sqle, i);
                conn.rollback();
            }
        }

        files = filesNf;

        addChangeset.setInt(1, reposId);

        // getHistoryEntries() returns the entries in reverse chronological
        // order, but we want to insert them in chronological order so that
        // their auto-generated identity column can be used as a chronological
        // ordering column. Otherwise, incremental updates will make the
        // identity column unusable for chronological ordering. So therefore
        // we walk the list backwards.
        for (ListIterator<HistoryEntry> it
                = entries.listIterator(entries.size());
                it.hasPrevious();) {
            HistoryEntry entry = it.previous();
            retry:
            for (int i = 0;; i++) {
                try {
                    addChangeset.setString(2, entry.getRevision());
                    addChangeset.setInt(3, authors.get(entry.getAuthor()));
                    addChangeset.setTimestamp(4,
                            new Timestamp(entry.getDate().getTime()));
                    String msg = entry.getMessage();
                    // Truncate the message if it can't fit in a VARCHAR
                    // (bug #11663).
                    if (msg.length() > MAX_MESSAGE_LENGTH) {
                        msg = truncate(msg, MAX_MESSAGE_LENGTH);
                    }
                    addChangeset.setString(5, msg);
                    int changesetId = nextChangesetId.getAndIncrement();
                    addChangeset.setInt(6, changesetId);
                    addChangeset.executeUpdate();

                    // Add one row for each file in FILECHANGES, and one row
                    // for each path element of the directories in DIRCHANGES.
                    Set<String> addedDirs = new HashSet<>();
                    addDirchange.setInt(1, changesetId);
                    addFilechange.setInt(1, changesetId);
                    for (String file : entry.getFiles()) {
                        // ignore non-existent files
                        String repodir = "";
                        try {
                            repodir = env.getPathRelativeToSourceRoot(
                                    new File(repository.getDirectoryName()), 0);
                        } catch (IOException ex) {
                            LOGGER.log(Level.WARNING,
                                    "File exception{0}", ex);
                            continue;
                        }

                        String fullPath = toUnixPath(file);
                        if (!history.isRenamed(
                                file.substring(repodir.length() + 1))
                                || !env.isHandleHistoryOfRenamedFiles()) {
                            int fileId = files.get(fullPath);
                            addFilechange.setInt(2, fileId);
                            addFilechange.executeUpdate();
                        }
                        String[] pathElts = splitPath(fullPath);
                        for (int j = 0; j < pathElts.length; j++) {
                            String dir = unsplitPath(pathElts, j);
                            // Only add to DIRCHANGES if we haven't already
                            // added this dir/changeset combination.
                            if (!addedDirs.contains(dir)) {
                                addDirchange.setInt(2, directories.get(dir));
                                addDirchange.executeUpdate();
                                addedDirs.add(dir);
                            }
                        }
                    }

                    conn.commit();

                    // Successfully added the entry. Break out of retry loop.
                    break retry;

                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                    conn.rollback();
                }
            }
        }

        if (!env.isHandleHistoryOfRenamedFiles()) {
            return;
        }

        /*
         * Special handling for certain files - this is mainly for files which
         * have been renamed in Mercurial repository.
         * This ensures that their complete history (follow) will be saved.
         */
        final CountDownLatch latch = new CountDownLatch(history.getRenamedFiles().size());
        for (String filename : history.getRenamedFiles()) {
            String file_path = repository.getDirectoryName()
                    + File.separatorChar + filename;
            final File file = new File(file_path);
            final String repo_path = file_path.substring(env.getSourceRootPath().length());

            RuntimeEnvironment.getHistoryRenamedExecutor().submit(new Runnable() {
                @Override
                public void run() {
                    try {
                        doRenamedHistory(repository, file, files, repo_path);
                    } catch (Exception ex) {
                        // We want to catch any exception since we are in thread.
                        LOGGER.log(Level.WARNING,
                                "doRenamedHistory exception {0}", ex);
                    } finally {
                        latch.countDown();
                    }
                }
            });
        }

        // Wait for the executors to finish.
        try {
            latch.await();
        } catch (InterruptedException ex) {
            LOGGER.log(Level.SEVERE,
                    "latch exception{0}", ex);
        }
        LOGGER.log(Level.FINE,
                "Done storing history for repo {0}",
                new Object[]{repository.getDirectoryName()});
    }

    /**
     * Optimize how the cache is stored on disk. In particular, make sure index
     * cardinality statistics are up to date, and perform a checkpoint to make
     * sure all changes are forced to the tables on disk and that the unneeded
     * transaction log is deleted.
     *
     * @throws HistoryException if an error happens when optimizing the cache
     */
    @Override
    public void optimize() throws HistoryException {
        try {
            final ConnectionResource conn
                    = connectionManager.getConnectionResource();
            try {
                updateIndexCardinalityStatistics(conn);
                checkpointDatabase(conn);
            } finally {
                connectionManager.releaseConnection(conn);
            }
        } catch (SQLException sqle) {
            throw new HistoryException(sqle);
        }
    }

    /**
     * <p>
     * Make sure Derby's index cardinality statistics are up to date. Otherwise,
     * the optimizer may choose a bad execution strategy for some queries. This
     * method should be called if the size of the tables has changed
     * significantly.
     * </p>
     *
     * <p>
     * This is a workaround for the problems described in
     * <a href="https://issues.apache.org/jira/browse/DERBY-269">DERBY-269</a>
     * and
     * <a href="https://issues.apache.org/jira/browse/DERBY-3788">DERBY-3788</a>.
     * When automatic update of index cardinality statistics has been
     * implemented in Derby, the workaround may be removed.
     * </p>
     *
     * <p>
     * Without this workaround, poor performance has been observed in
     * {@code get()} due to bad choices made by the optimizer.
     * </p>
     *
     * <p>
     * Note that this method uses a system procedure introduced in Derby 10.5.
     * If this procedure does not exist, this method is a no-op.
     * </p>
     */
    private void updateIndexCardinalityStatistics(ConnectionResource conn)
            throws SQLException {
        DatabaseMetaData dmd = conn.getMetaData();
        if (procedureExists(dmd, "SYSCS_UTIL", "SYSCS_UPDATE_STATISTICS")) {
            try (PreparedStatement ps = conn.prepareStatement(
                    "CALL SYSCS_UTIL.SYSCS_UPDATE_STATISTICS(?, ?, NULL)")) {
                ps.setString(1, SCHEMA);
                for (String table : TABLES) {
                    ps.setString(2, table);
                    retry:
                    for (int i = 0;; i++) {
                        try {
                            ps.execute();
                            // Successfully executed statement. Break out of
                            // retry loop.
                            break retry;
                        } catch (SQLException sqle) {
                            handleSQLException(sqle, i);
                            conn.rollback();
                        }
                    }
                    conn.commit();
                }
            }
        }
    }

    /**
     * If this is a Derby database, force a checkpoint so that the disk space
     * occupied by the transaction log is freed as early as possible.
     */
    private void checkpointDatabase(ConnectionResource conn)
            throws SQLException {
        DatabaseMetaData dmd = conn.getMetaData();
        if (procedureExists(dmd, "SYSCS_UTIL", "SYSCS_CHECKPOINT_DATABASE")) {
            try (Statement s = conn.createStatement()) {
                s.execute("CALL SYSCS_UTIL.SYSCS_CHECKPOINT_DATABASE()");
            }
            conn.commit();
        }
    }

    /**
     * Check if a stored database procedure exists.
     *
     * @param dmd the meta-data object used for checking
     * @param schema the procedure's schema
     * @param proc the name of the procedure
     * @return {@code true} if the procedure exists, {@code false} otherwise
     * @throws SQLException if an error happens when reading the meta-data
     */
    private boolean procedureExists(DatabaseMetaData dmd,
            String schema, String proc)
            throws SQLException {
        try (ResultSet rs = dmd.getProcedures(null, schema, proc)) {
            // If there's a row, there is such a procedure.
            return rs.next();
        }
    }

    /**
     * Get the id of a repository in the database. If the repository is not
     * stored in the database, add it and return its id.
     *
     * @param conn the connection to the database
     * @param repository the repository whose id to get
     * @return the id of the repository
     */
    private int getRepositoryId(ConnectionResource conn, Repository repository)
            throws SQLException {
        String reposPath = toUnixPath(repository.getDirectoryName());
        PreparedStatement reposIdPS = conn.getStatement(GET_REPOSITORY);
        reposIdPS.setString(1, reposPath);
        try (ResultSet reposIdRS = reposIdPS.executeQuery()) {
            if (reposIdRS.next()) {
                return reposIdRS.getInt(1);
            }
        }

        // Repository is not in the database. Add it.
        PreparedStatement insert
                = conn.getStatement(INSERT_REPOSITORY);
        insert.setString(1, reposPath);
        insert.executeUpdate();
        return getGeneratedIntKey(insert);
    }

    /**
     * Get a map from author names to their ids in the database. The authors
     * that are not in the database are added to it.
     *
     * @param conn the connection to the database
     * @param history the history to get the author names from
     * @param reposId the id of the repository
     * @return a map from author names to author ids
     */
    private Map<String, Integer> getAuthors(
            ConnectionResource conn, History history, int reposId)
            throws SQLException {
        HashMap<String, Integer> map = new HashMap<>();
        PreparedStatement ps = conn.getStatement(GET_AUTHORS);
        ps.setInt(1, reposId);
        try (ResultSet rs = ps.executeQuery()) {
            while (rs.next()) {
                map.put(rs.getString(1), rs.getInt(2));
            }
        }

        PreparedStatement insert = conn.getStatement(ADD_AUTHOR);
        insert.setInt(1, reposId);
        for (HistoryEntry entry : history.getHistoryEntries()) {
            String author = entry.getAuthor();
            if (!map.containsKey(author)) {
                int id = nextAuthorId.getAndIncrement();
                insert.setString(2, author);
                insert.setInt(3, id);
                insert.executeUpdate();
                map.put(author, id);
                conn.commit();
            }
        }

        return map;
    }

    /**
     * Build maps from directory names and file names to their respective
     * identifiers in the database. The directories and files that are not
     * already in the database, are added to it.
     *
     * @param conn the connection to the database
     * @param history the history to get the file and directory names from
     * @param reposId the id of the repository
     * @param dirMap a map which will be filled with directory names and ids
     * @param fileMap a map which will be filled with file names and ids
     */
    private void getFilesAndDirectories(
            ConnectionResource conn, History history, int reposId,
            Map<String, Integer> dirMap, Map<String, Integer> fileMap)
            throws SQLException {

        populateFileOrDirMap(conn.getStatement(GET_DIRS), reposId, dirMap);
        populateFileOrDirMap(conn.getStatement(GET_FILES), reposId, fileMap);

        int insertCount = 0;

        PreparedStatement insDir = conn.getStatement(INSERT_DIR);
        PreparedStatement insFile = conn.getStatement(INSERT_FILE);
        for (HistoryEntry entry : history.getHistoryEntries()) {
            for (String file : entry.getFiles()) {
                String fullPath = toUnixPath(file);
                // Add the file to the database and to the map if it isn't
                // there already. Assumption: If the file is in the database,
                // all its parent directories are also there.
                if (!fileMap.containsKey(fullPath)) {
                    // Get the dir id for this file, potentially adding the
                    // parent directories to the db and to dirMap.
                    int dir = addAllDirs(insDir, reposId, fullPath, dirMap);
                    int fileId = nextFileId.getAndIncrement();
                    insFile.setInt(1, dir);
                    insFile.setString(2, getBaseName(fullPath));
                    insFile.setInt(3, fileId);
                    insFile.executeUpdate();
                    fileMap.put(fullPath, fileId);

                    // Commit every now and then to allow the database to free
                    // resources (like locks and transaction log), but not too
                    // frequently, since that may kill the performance. It is
                    // OK not to commit for every file added, since the worst
                    // thing that could happen is that we need to re-insert
                    // the files added since the last commit in case of a crash.
                    insertCount++;
                    if (insertCount % 30 == 0) {
                        conn.commit();
                    }
                }
            }
        }
    }

    /**
     * Populate a map with all path/id combinations found in the FILES or
     * DIRECTORIES tables associated with a specified repository id.
     *
     * @param ps the statement used to get path names and ids from the correct
     * table. It should take one parameter: the repository id.
     * @param reposId the id of the repository to scan
     * @param map the map into which to insert the path/id combinations
     */
    private void populateFileOrDirMap(
            PreparedStatement ps, int reposId, Map<String, Integer> map)
            throws SQLException {
        ps.setInt(1, reposId);
        try (ResultSet rs = ps.executeQuery()) {
            while (rs.next()) {
                map.put(rs.getString(1), rs.getInt(2));
            }
        }
    }

    /**
     * Add all the parent directories of a specified file to the database, if
     * they haven't already been added, and also put their paths and ids into a
     * map.
     *
     * @param ps statement that inserts a directory into the DIRECTORY table.
     * Takes three parameters: (1) the id of the repository, (2) the path of the
     * directory, and (3) the id to use for the directory.
     * @param reposId id of the repository to which the file belongs
     * @param fullPath the file whose parents to add
     * @param map a map from directory path to id for the directories already in
     * the database. When a new directory is added, it's also added to this map.
     * @return the id of the first parent of {@code fullPath}
     */
    private int addAllDirs(
            PreparedStatement ps, int reposId, String fullPath,
            Map<String, Integer> map) throws SQLException {
        String[] pathElts = splitPath(fullPath);
        String parent = unsplitPath(pathElts, pathElts.length - 1);
        Integer dir = map.get(parent);
        if (dir == null) {
            for (int i = 0; i < pathElts.length; i++) {
                Integer prevDirId = dir;
                String path = unsplitPath(pathElts, i);
                dir = map.get(path);
                if (dir == null) {
                    dir = nextDirId.getAndIncrement();
                    ps.setInt(1, reposId);
                    ps.setString(2, path);
                    ps.setInt(3, dir);
                    ps.setObject(4, prevDirId, Types.INTEGER);
                    ps.executeUpdate();
                    map.put(path, dir);
                }
            }
        }
        return dir;
    }

    /**
     * Return the integer key generated by the previous execution of a
     * statement. The key should be a single INTEGER, and the statement should
     * insert exactly one row, so there should be only one key.
     *
     * @param stmt a statement that has just inserted a row
     * @return the integer key for the newly inserted row, or {@code null} if
     * there is no key
     */
    private Integer getGeneratedIntKey(Statement stmt) throws SQLException {
        try (ResultSet keys = stmt.getGeneratedKeys()) {
            return keys.next() ? keys.getInt(1) : null;
        }
    }

    @Override
    public String getLatestCachedRevision(Repository repository)
            throws HistoryException {
        try {
            for (int i = 0;; i++) {
                try {
                    return getLatestRevisionForRepository(repository);
                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                }
            }
        } catch (SQLException sqle) {
            throw new HistoryException(sqle);
        }
    }

    /**
     * Helper for {@link #getLatestCachedRevision(Repository)}.
     */
    private String getLatestRevisionForRepository(Repository repository)
            throws SQLException {
        final ConnectionResource conn
                = connectionManager.getConnectionResource();
        try {
            PreparedStatement ps = conn.getStatement(GET_LATEST_REVISION);
            ps.setString(1, toUnixPath(repository.getDirectoryName()));
            try (ResultSet rs = ps.executeQuery()) {
                return rs.next() ? rs.getString(1) : null;
            }
        } finally {
            connectionManager.releaseConnection(conn);
        }
    }

    @Override
    public Map<String, Date> getLastModifiedTimes(
            File directory, Repository repository)
            throws HistoryException {
        try {
            for (int i = 0;; i++) {
                try {
                    return getLastModifiedTimesForAllFiles(
                            directory, repository);
                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                }
            }
        } catch (SQLException sqle) {
            throw new HistoryException(sqle);
        }
    }

    private Map<String, Date> getLastModifiedTimesForAllFiles(
            File directory, Repository repository)
            throws HistoryException, SQLException {
        final Map<String, Date> map = new HashMap<>();

        final ConnectionResource conn
                = connectionManager.getConnectionResource();
        try {
            PreparedStatement ps = conn.getStatement(GET_LAST_MODIFIED_TIMES);
            ps.setString(1, toUnixPath(repository.getDirectoryName()));
            ps.setString(2, getSourceRootRelativePath(directory));
            try (ResultSet rs = ps.executeQuery()) {
                while (rs.next()) {
                    map.put(rs.getString(1), rs.getTimestamp(2));
                }
            }
        } finally {
            connectionManager.releaseConnection(conn);
        }

        return map;
    }

    @Override
    public void clear(Repository repository) throws HistoryException {
        try {
            for (int i = 0;; i++) {
                try {
                    clearHistoryForRepository(repository);
                    return;
                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                }
            }
        } catch (SQLException sqle) {
            throw new HistoryException(sqle);
        }
    }

    /**
     * Helper for {@link #clear(Repository)}.
     */
    private void clearHistoryForRepository(Repository repository)
            throws SQLException {
        final ConnectionResource conn
                = connectionManager.getConnectionResource();
        try {
            try (PreparedStatement ps = conn.prepareStatement(
                    getQuery("clearRepository"))) {
                ps.setInt(1, getRepositoryId(conn, repository));
                ps.execute();
                conn.commit();
            }
        } finally {
            connectionManager.releaseConnection(conn);
        }
    }

    @Override
    public String getInfo() {
        return info;
    }

    /*
     * Create history cache for file which has been renamed in the past.
     * This inserts data both into FILECHANGES and FILEMOVES tables.
     */
    private void doRenamedHistory(final Repository repository, File file,
            Map<String, Integer> files, String repo_path)
            throws SQLException {
        History hist;
        PreparedStatement addFilemove;
        PreparedStatement addFilechange;

        try {
            hist = repository.getHistory(file);
        } catch (HistoryException ex) {
            LOGGER.log(Level.WARNING,
                    "cannot get history for {0} because of exception {1}",
                    new Object[]{file, ex});
            return;
        }

        int fileId = files.get(repo_path);
        for (HistoryEntry entry : hist.getHistoryEntries()) {
            retry:
            for (int i = 0;; i++) {

                final ConnectionResource conn
                        = connectionManager.getConnectionResource();

                addFilemove = conn.getStatement(ADD_FILEMOVE);
                addFilechange = conn.getStatement(ADD_FILECHANGE);

                try {
                    int changesetId = getIdForRevision(entry.getRevision(),
                            getRepositoryId(conn, repository));

                    /*
                     * If the file exists in the changeset, store it in
                     * the table tracking moves of the file when it had
                     * one of its precedent names so it can be found
                     * when performing historyget on directory.
                     */
                    if (entry.getFiles().contains(repo_path)) {
                        addFilechange.setInt(1, changesetId);
                        addFilechange.setInt(2, fileId);
                        addFilechange.executeUpdate();
                    } else {
                        addFilemove.setInt(1, changesetId);
                        addFilemove.setInt(2, fileId);
                        addFilemove.executeUpdate();
                    }

                    conn.commit();
                    break retry;
                } catch (SQLException sqle) {
                    handleSQLException(sqle, i);
                    conn.rollback();
                } finally {
                    connectionManager.releaseConnection(conn);
                }
            }
        }
    }

    @Override
    public void setHistoryIndexDone() {
        historyIndexDone = true;
    }

    @Override
    public boolean isHistoryIndexDone() {
        return historyIndexDone;
    }
}