backup revision a8028fd2789e323040de08827a0fe1f7d36fde2b
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# You can find more extensive documentation of this script at
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# https://github.com/ontohub/ontohub/blob/staging/doc/backup_and_restore_of_ontohub_data.md
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# Description
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# This backup script creates and restores backups of ontohub data. It includes:
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# * bare git repositories (data/repositories)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# * named symlinks to git repositories (data/git_daemon)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# * the postgres database
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# First note: Run this as the ontohub user, *not* as root.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# To create a backup, run this script with the argument `create`:
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# Then a backup named with the current date and time is created in the
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# backup directory (see below).
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# To restore a backup, run this script with the argument `restore <backup name>`
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# $ script/backup restore 2015-01-01_00-00
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# Then the selected backup is fully restored
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# Backup directory
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# For development machines, the backup directory is:
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# <rails root>/tmp/backup/
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# And for production machines, the backup directory is:
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# Super user privileges
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# To create and restore, we need root privileges. Otherwise file modes are not
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# preserved. This script will call `sudo` when needed and inform you about the
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# reason for calling `sudo`. If you don't allow sudo, a backup will be created
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# or restored anyway, but the file modes and ownership are not preserved.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# Then, you need to adjust them manually.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# Maintenance mode
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# While backing up and restoring the data, the maintenance mode is activated.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# This way we guarantee data consistency of the backup.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncrequire 'fileutils'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncrequire 'pathname'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncrequire 'open3'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync # Amount of backups that have to be there at least
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync # Backups are kept for at least 365 days
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync attr_reader :dry_run, :verbose, :sql_dump_as_postgres_user
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync verbose: false, dry_run: true, sql_dump_as_postgres_user: false)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync @data_dirs = DATA_DIRS.map { |dir| File.join(@data_root_basename, dir) }
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync @sql_dump_as_postgres_user = sql_dump_as_postgres_user
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync # We needed to create the directory for the script to continue later on.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts "Nothing to prune: There is no backup directory."
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync @backup_instance_dir = backup_root.join(new_backup_name)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync # Create directory even in dry run to let the script continue.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync exec('pg_dump', *pg_user_switch, '-Fc', db_name, '-f', SQL_DUMP_FILE)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync archive_file = backup_instance_dir.join(REPOSITORY_FILE)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync exec('tar', verbose ? '-v' : '', '-cf', archive_file.to_s, *@data_dirs)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync @backup_instance_dir = backup_root.join(backup_name)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync "Error: Backup '#{backup_name}' does not exist in #{backup_root}.")
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync exec('pg_restore', '-c', *pg_user_switch, '-d', db_name, SQL_DUMP_FILE)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncAn error occured while restoring the repositories:
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync#{e.message}
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncYou can find the pre-restore repositories at #{tmpdir}
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncDo something about it.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def move_data_dirs_to_tmpdir(tmpdir)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts "FileUtils.mv(#{@data_dirs}, #{tmpdir})" if verbose
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync FileUtils.mv(@data_dirs, tmpdir) unless dry_run
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync rescue Errno::EACCES
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts <<-MSG
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncAs the current user I have no access to move the repository data
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncdirectories #{@data_dirs.join(' ')} to a temporary directory #{tmpdir}.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncThis is used as a backup for the case of an error while restoring.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncTo continue, I try the command again using sudo.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync exec('sudo', 'mv', *@data_dirs, tmpdir)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def extract_archive
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync archive_file = backup_instance_dir.join(REPOSITORY_FILE)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts <<-MSG
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncSuper user privileges are needed to reset the file permissions as
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncthey were before the backup. If you refuse to enter the password
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync(Ctl-C) or enter a wrong password, only the permissions will not be
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncrestored and all restored files will belong to the current user/group.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync try_as_sudo_with_fallback('tar', verbose ? '-v' : '', '-xf',
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync archive_file.to_s, *@data_dirs)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def remove_tmpdir(tmpdir)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts "FileUtils.remove_entry(#{tmpdir})" if verbose
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync FileUtils.remove_entry(tmpdir) # even do this in dry run
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync rescue Errno::EACCES
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts <<-MSG
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncAs the current user I have no access to remove the temporary
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncdirectory #{tmpdir}.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncTo continue, I try the command again using sudo.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync exec('sudo', 'rm', '-r', tmpdir)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def enable_maintenance_mode
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts 'Enabling maintenance mode...'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync if File.exist?(maintenance_file)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts 'Maintenance mode was already enabled.'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts "Please check the file #{maintenance_file}"
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts 'Aborting.'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts "FileUtils.touch #{maintenance_file}" if verbose
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync FileUtils.touch maintenance_file unless dry_run
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def disable_maintenance_mode
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts 'Disabling maintenance mode...'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts "FileUtils.rm #{maintenance_file}" if verbose
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync FileUtils.rm maintenance_file unless dry_run
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def exec(*args)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts "[executing next command in #{Dir.getwd}]" if verbose
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync out = args.join(' ')
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts out if verbose
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync system(*args) unless dry_run
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def try_as_sudo_with_fallback(*args)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync _out, _err, exit_code = exec('sudo', *args)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync unless exit_code.success?
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync sudo_not_given_fallback(*args) # Wrong sudo password
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync rescue Exception => e
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync raise e unless e.is_a?(Interrupt) # Ctrl-C when asked for password
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync sudo_not_given_fallback(*args)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def sudo_not_given_fallback(*args)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts 'Super user privileges not granted. Trying as normal user.'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync exec(*args)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def maintenance_file
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync data_root.join(MAINTENANCE_FILE)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def pg_user_switch
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync sql_dump_as_postgres_user ? %w(-U postgres) : []
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync def self.backup_dirs_allowed_to_delete(entries)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncdef data_root(rails_root)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync if on_development_system?(rails_root)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync File.realpath(rails_root.join('data'))
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync ENV['DATA_ROOT'] ||'/data/git'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncdef on_development_system?(rails_root)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync data_path = rails_root.join('data')
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync File.exist?(data_path) && !File.symlink?(data_path)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# Don't allow this to be run as the root user.
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncif ENV['USER'] == 'root'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts 'Running this script as the root user is disabled.'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync puts 'Please run it as a normal user that has sudo privileges, e.g. ontohub.'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync# We assume, this script runs in "RAILS_ROOT/script/".
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncRAILS_ROOT = Pathname.new(__FILE__).dirname.join('..')
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncBACKUP_ROOT_PRODUCTION = '/local/home/ontohub/ontohub_data_backup'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync if on_development_system?(RAILS_ROOT)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync 'ontohub_development'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncBACKUP_ROOT =
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync if on_development_system?(RAILS_ROOT)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync RAILS_ROOT.join('tmp', 'backup')
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync File.realpath(BACKUP_ROOT_PRODUCTION)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncbackup = Backup::Backup.new(DATABASE, data_root(RAILS_ROOT), BACKUP_ROOT,
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync sql_dump_as_postgres_user: on_development_system?(RAILS_ROOT),
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync dry_run: false, verbose: true)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsynccase ARGV.first
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncwhen 'create'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync backup.create
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncwhen 'restore'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync if ARGV.length == 1
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts(
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync 'To restore a backup, you need to specify one with the arguments')
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts('"restore backup_name"')
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync backup_name = ARGV[1]
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync backup.restore(backup_name)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsyncwhen 'prune'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync Backup::Backup.prune(BACKUP_ROOT)
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts 'unknown or missing parameter'
3194da424708abdd288b28d96892b3a5f3f7df0bvboxsync $stderr.puts 'use parameter "create" or "restore <backup_name>" or "prune"'