Cross Reference: /ontohub/script/backup

	backup revision bacf95fddc53ab3107f380c4c816fa8072358bd9
a466f24ecf94b453750f9791ad1140556114711bvboxsync#!/usr/bin/env ruby
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync# Description
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   This backup script creates and restores backups of ontohub data. It includes:
a466f24ecf94b453750f9791ad1140556114711bvboxsync#     * bare git repositories (data/repositories)
c97989161fbe75bc14cea477a5443bbf474dd3advboxsync#     * named symlinks to git repositories (data/git_daemon)
a466f24ecf94b453750f9791ad1140556114711bvboxsync#     * the postgres database
a466f24ecf94b453750f9791ad1140556114711bvboxsync#
a466f24ecf94b453750f9791ad1140556114711bvboxsync# Usage
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   First note: Run this as the ontohub user, *not* as root.
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   To create a backup, run this script with the argument `create`:
a466f24ecf94b453750f9791ad1140556114711bvboxsync#     $ script/backup create
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   Then a backup named with the current date and time is created in the
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   backup directory (see below).
a466f24ecf94b453750f9791ad1140556114711bvboxsync#
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   To restore a backup, run this script with the argument `restore <backup name>`
a466f24ecf94b453750f9791ad1140556114711bvboxsync#     $ script/backup restore 2015-01-01_00-00
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   Then the selected backup is fully restored
a466f24ecf94b453750f9791ad1140556114711bvboxsync#
a466f24ecf94b453750f9791ad1140556114711bvboxsync# Backup directory
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   For development machines, the backup directory is:
a466f24ecf94b453750f9791ad1140556114711bvboxsync#     <rails root>/tmp/backup/
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   And for production machines, the backup directory is:
a466f24ecf94b453750f9791ad1140556114711bvboxsync#     /home/ontohub/ontohub_data_backup
a466f24ecf94b453750f9791ad1140556114711bvboxsync#
a466f24ecf94b453750f9791ad1140556114711bvboxsync# Super user privileges
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   To create and restore, we need root privileges. Otherwise file modes are not
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   preserved. This script will call `sudo` when needed and inform you about the
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   reason for calling `sudo`. If you don't allow sudo, a backup will be created
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   or restored anyway, but the file modes and ownership are not preserved.
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   Then, you need to adjust them manually.
a466f24ecf94b453750f9791ad1140556114711bvboxsync#
a466f24ecf94b453750f9791ad1140556114711bvboxsync# Maintenance mode
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   While backing up and restoring the data, the maintenance mode is activated.
a466f24ecf94b453750f9791ad1140556114711bvboxsync#   This way we guarantee data consistency of the backup.
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsyncrequire 'tmpdir.rb'
a466f24ecf94b453750f9791ad1140556114711bvboxsyncrequire 'fileutils'
a466f24ecf94b453750f9791ad1140556114711bvboxsyncrequire 'pathname'
a466f24ecf94b453750f9791ad1140556114711bvboxsyncrequire 'open3'
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsyncmodule Backup
a466f24ecf94b453750f9791ad1140556114711bvboxsync  class Backup
a466f24ecf94b453750f9791ad1140556114711bvboxsync    # Amount of backups that have to be there at least
a466f24ecf94b453750f9791ad1140556114711bvboxsync    BACKUPS_COUNT = 30
a466f24ecf94b453750f9791ad1140556114711bvboxsync    # Backups are kept for at least 365 days
a466f24ecf94b453750f9791ad1140556114711bvboxsync    BACKUPS_VALIDITY_TIME = 365 * 60 * 60 * 24
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    MAINTENANCE_FILE = 'maintenance.txt'
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    SQL_DUMP_FILE = 'ontohub_sql_dump.postgresql'
a466f24ecf94b453750f9791ad1140556114711bvboxsync    REPOSITORY_FILE = 'ontohub_repositories.tar.gz'
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    DATA_DIRS = %w(data/repositories data/git_daemon)
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
a466f24ecf94b453750f9791ad1140556114711bvboxsync    attr_reader :dry_run, :verbose, :sql_dump_as_postgres_user
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def initialize(db_name, data_root, backup_root,
a466f24ecf94b453750f9791ad1140556114711bvboxsync      verbose: false, dry_run: true, sql_dump_as_postgres_user: false)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @db_name = db_name
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @data_root = Pathname.new(data_root)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @backup_root = Pathname.new(backup_root)
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @dry_run = dry_run
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @verbose = verbose
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @sql_dump_as_postgres_user = sql_dump_as_postgres_user
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def create
a466f24ecf94b453750f9791ad1140556114711bvboxsync      puts 'Creating backup...'
a466f24ecf94b453750f9791ad1140556114711bvboxsync      enable_maintenance_mode
a466f24ecf94b453750f9791ad1140556114711bvboxsync      initialize_backup
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      create_sql_dump
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      create_repository_archive
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      # We needed to create the directory for the script to continue later on.
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      Dir.rmdir(backup_instance_dir) if dry_run
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      disable_maintenance_mode
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      puts "Created backup in #{backup_instance_dir}"
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      self.class.prune(backup_root)
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync    end
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync    def restore(backup_name)
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      enable_maintenance_mode
eb270e8b46c0ad30cdffc6b75fc8dffef9738fbdvboxsync      initialize_restore(backup_name)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      restore_sql_dump
a466f24ecf94b453750f9791ad1140556114711bvboxsync      restore_repository_archive
a466f24ecf94b453750f9791ad1140556114711bvboxsync      disable_maintenance_mode
a466f24ecf94b453750f9791ad1140556114711bvboxsync      puts "Restored backup from #{backup_instance_dir}"
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def self.prune(backup_root)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      if !Dir.exists?(backup_root)
a466f24ecf94b453750f9791ad1140556114711bvboxsync        $stderr.puts "Nothing to prune: There is no backup directory."
a466f24ecf94b453750f9791ad1140556114711bvboxsync        return
a466f24ecf94b453750f9791ad1140556114711bvboxsync      end
a466f24ecf94b453750f9791ad1140556114711bvboxsync      now = Time.now
a466f24ecf94b453750f9791ad1140556114711bvboxsync      backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
a466f24ecf94b453750f9791ad1140556114711bvboxsync        backup = backup_root.join(dir)
a466f24ecf94b453750f9791ad1140556114711bvboxsync        if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
a466f24ecf94b453750f9791ad1140556114711bvboxsync          puts "removing old backup: #{dir}"
a466f24ecf94b453750f9791ad1140556114711bvboxsync          FileUtils.rm_r(backup)
a466f24ecf94b453750f9791ad1140556114711bvboxsync        end
a466f24ecf94b453750f9791ad1140556114711bvboxsync      end
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    protected
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def new_backup_name
a466f24ecf94b453750f9791ad1140556114711bvboxsync      Time.now.strftime("%Y-%m-%d_%H-%M-%S")
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def initialize_backup
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @backup_instance_dir = backup_root.join(new_backup_name)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
a466f24ecf94b453750f9791ad1140556114711bvboxsync      # Create directory even in dry run to let the script continue.
a466f24ecf94b453750f9791ad1140556114711bvboxsync      FileUtils.mkdir_p(backup_instance_dir)
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def create_sql_dump
fbce7e4e7123ca200586ee9257a12d663e928841vboxsync      puts 'Creating SQL dump...'
a466f24ecf94b453750f9791ad1140556114711bvboxsync      Dir.chdir(backup_instance_dir) do
a466f24ecf94b453750f9791ad1140556114711bvboxsync        exec('pg_dump', *pg_user_switch, '-Fc', db_name, '-f', SQL_DUMP_FILE)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      end
09624e5df01e54dc2df112e929c91145768446bdvboxsync    end
fbce7e4e7123ca200586ee9257a12d663e928841vboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def create_repository_archive
a466f24ecf94b453750f9791ad1140556114711bvboxsync      puts 'Creating repository archive...'
a466f24ecf94b453750f9791ad1140556114711bvboxsync      Dir.chdir(data_root.join('..')) do
fec6782953105e154c193c1be4a761da1c148d6fvboxsync        archive_file = backup_instance_dir.join(REPOSITORY_FILE)
11ab86db9b44d06e180fbf806c9d1915be136d1evboxsync        exec('tar', verbose ? '-v' : '', '-cf', archive_file.to_s,
a466f24ecf94b453750f9791ad1140556114711bvboxsync          *DATA_DIRS.map(&:to_s))
a466f24ecf94b453750f9791ad1140556114711bvboxsync      end
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def initialize_restore(backup_name)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      @backup_instance_dir = backup_root.join(backup_name)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      unless Dir.exists?(backup_instance_dir)
a466f24ecf94b453750f9791ad1140556114711bvboxsync        $stderr.puts (
a466f24ecf94b453750f9791ad1140556114711bvboxsync          "Error: Backup '#{backup_name}' does not exist in #{backup_root}.")
a466f24ecf94b453750f9791ad1140556114711bvboxsync        exit
a466f24ecf94b453750f9791ad1140556114711bvboxsync      end
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def restore_sql_dump
a466f24ecf94b453750f9791ad1140556114711bvboxsync      'Restoring SQL dump...'
a466f24ecf94b453750f9791ad1140556114711bvboxsync      Dir.chdir(backup_instance_dir) do
a466f24ecf94b453750f9791ad1140556114711bvboxsync        exec('pg_restore', '-c', *pg_user_switch, '-d', db_name, SQL_DUMP_FILE)
a466f24ecf94b453750f9791ad1140556114711bvboxsync      end
a466f24ecf94b453750f9791ad1140556114711bvboxsync    end
a466f24ecf94b453750f9791ad1140556114711bvboxsync
a466f24ecf94b453750f9791ad1140556114711bvboxsync    def restore_repository_archive
a466f24ecf94b453750f9791ad1140556114711bvboxsync      puts 'Restoring repository archive...'
a466f24ecf94b453750f9791ad1140556114711bvboxsync      Dir.chdir(data_root.join('..')) do
        tmpdir = Dir.mktmpdir
        move_data_dirs_to_tmpdir(tmpdir)
        begin
          extract_archive
          remove_tmpdir(tmpdir)
        rescue => e
          puts <<-MSG

An error occured while restoring the repositories:
#{e.message}
You can find the pre-restore repositories at #{tmpdir}
Do something about it.
          MSG
          raise e
        end
      end
    end

    def move_data_dirs_to_tmpdir(tmpdir)
      puts "FileUtils.mv(#{DATA_DIRS}, #{tmpdir})" if verbose
      FileUtils.mv(DATA_DIRS, tmpdir) unless dry_run
    rescue Errno::EACCES
      puts <<-MSG

As the current user I have no access to move the repository data
directories #{DATA_DIRS.join(' ')} to a temporary directory #{tmpdir}.
This is used as a backup for the case of an error while restoring.
To continue, I try the command again using sudo.
      MSG
      exec('sudo', 'mv', *DATA_DIRS.map(&:to_s), tmpdir)
    end

    def extract_archive
      archive_file = backup_instance_dir.join(REPOSITORY_FILE)
      puts <<-MSG

Super user privileges are needed to reset the file permissions as
they were before the backup. If you refuse to enter the password
(Ctl-C) or enter a wrong password, only the permissions will not be
restored and all restored files will belong to the current user/group.
      MSG
      try_as_sudo_with_fallback('tar', verbose ? '-v' : '', '-xf',
        archive_file.to_s, *DATA_DIRS.map(&:to_s))
    end

    def remove_tmpdir(tmpdir)
      puts "FileUtils.remove_entry(#{tmpdir})" if verbose
      FileUtils.remove_entry(tmpdir) # even do this in dry run
    rescue Errno::EACCES
      puts <<-MSG

As the current user I have no access to remove the temporary
directory #{tmpdir}.
To continue, I try the command again using sudo.
      MSG
      exec('sudo', 'rm', '-r', tmpdir)
    end

    def enable_maintenance_mode
      puts 'Enabling maintenance mode...'
      if File.exist?(maintenance_file)
        $stderr.puts 'Maintenance mode was already enabled.'
        $stderr.puts "Please check the file #{maintenance_file}"
        $stderr.puts 'Aborting.'
        exit
      end
      puts "FileUtils.touch #{maintenance_file}" if verbose
      FileUtils.touch maintenance_file unless dry_run
    end

    def disable_maintenance_mode
      puts 'Disabling maintenance mode...'
      puts "FileUtils.rm #{maintenance_file}" if verbose
      FileUtils.rm maintenance_file unless dry_run
    end

    def exec(*args)
      puts "[executing next command in #{Dir.getwd}]" if verbose
      out = args.join(' ')
      puts out if verbose
      system(*args) unless dry_run
    end

    def try_as_sudo_with_fallback(*args)
      _out, _err, exit_code = exec('sudo', *args)
      unless exit_code.success?
        sudo_not_given_fallback(*args) # Wrong sudo password
      end
    rescue Exception => e
      raise e unless e.is_a?(Interrupt) # Ctrl-C when asked for password
      sudo_not_given_fallback(*args)
    end

    def sudo_not_given_fallback(*args)
      puts 'Super user privileges not granted. Trying as normal user.'
      exec(*args)
    end

    def maintenance_file
      data_root.join(MAINTENANCE_FILE)
    end

    def pg_user_switch
      sql_dump_as_postgres_user ? %w(-U postgres) : []
    end

    def self.backup_dirs_allowed_to_delete(entries)
      entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
    end
  end
end

def data_root(rails_root)
  File.realpath(rails_root.join('data'))
end

def on_development_system?(rails_root)
  !File.symlink?(rails_root.join('data'))
end

# Don't allow this to be run as the root user.
if ENV['USER'] == 'root'
  puts 'Running this script as the root user is disabled.'
  puts 'Please run it as a normal user that has sudo privileges, e.g. ontohub.'
  exit
end

# We assume, this script runs in "RAILS_ROOT/script/".
RAILS_ROOT = Pathname.new(__FILE__).dirname.join('..')
BACKUP_ROOT_PRODUCTION = '/home/ontohub/ontohub_data_backup'

DATABASE =
  if on_development_system?(RAILS_ROOT)
    'ontohub_development'
  else
    'ontohub'
  end

BACKUP_ROOT =
  if on_development_system?(RAILS_ROOT)
    RAILS_ROOT.join('tmp', 'backup')
  else
    File.realpath(BACKUP_ROOT_PRODUCTION)
  end

backup = Backup::Backup.new(DATABASE, data_root(RAILS_ROOT), BACKUP_ROOT,
  sql_dump_as_postgres_user: on_development_system?(RAILS_ROOT),
  dry_run: false, verbose: true)

case ARGV.first
when 'create'
  backup.create
when 'restore'
  if ARGV.length == 1
    $stderr.puts(
      'To restore a backup, you need to specify one with the arguments')
    $stderr.puts('"restore backup_name"')
    exit
  end
  backup_name = ARGV[1]
  backup.restore(backup_name)
when 'prune'
  Backup::Backup.prune(BACKUP_ROOT)
else
  $stderr.puts 'unknown or missing parameter'
  $stderr.puts 'use parameter "create" or "restore <backup_name>" or "prune"'
  exit
end