backup revision f9e467b8fe6fef705eec2989b20e92eaa9d917e1
3387N/A#!/usr/bin/env ruby
3387N/A
3387N/A# You can find more extensive documentation of this script at
3387N/A# https://github.com/ontohub/ontohub/blob/staging/doc/backup_and_restore_of_ontohub_data.md
3387N/A
3387N/A# Description
3387N/A# This backup script creates and restores backups of ontohub data. It includes:
3387N/A# * bare git repositories (data/repositories)
3387N/A# * named symlinks to git repositories (data/git_daemon)
3387N/A# * the postgres database
3387N/A#
3387N/A# Usage
3387N/A# First note: Run this as the ontohub user, *not* as root.
3387N/A# To create a backup, run this script with the argument `create`:
3387N/A# $ script/backup create
3387N/A# Then a backup named with the current date and time is created in the
3387N/A# backup directory (see below).
3387N/A#
3387N/A# To restore a backup, run this script with the argument `restore <backup name>`
3387N/A# $ script/backup restore 2015-01-01_00-00
3387N/A# Then the selected backup is fully restored
3387N/A#
3387N/A# Backup directory
3387N/A# For development machines, the backup directory is:
3387N/A# <rails root>/tmp/backup/
3387N/A# And for production machines, the backup directory is:
3387N/A# /home/ontohub/ontohub_data_backup
3387N/A#
3387N/A# Super user privileges
3387N/A# To create and restore, we need root privileges. Otherwise file modes are not
3387N/A# preserved. This script will call `sudo` when needed and inform you about the
3387N/A# reason for calling `sudo`. If you don't allow sudo, a backup will be created
3387N/A# or restored anyway, but the file modes and ownership are not preserved.
3387N/A# Then, you need to adjust them manually.
3387N/A#
3387N/A# Maintenance mode
3387N/A# While backing up and restoring the data, the maintenance mode is activated.
3387N/A# This way we guarantee data consistency of the backup.
3387N/A
3387N/A
3387N/Arequire 'tmpdir.rb'
3387N/Arequire 'fileutils'
3387N/Arequire 'pathname'
3387N/Arequire 'open3'
3387N/A
3387N/Amodule Backup
3387N/A class Backup
3387N/A # Amount of backups that have to be there at least
3387N/A BACKUPS_COUNT = 30
3387N/A # Backups are kept for at least 365 days
3387N/A BACKUPS_VALIDITY_TIME = 365 * 60 * 60 * 24
3387N/A
3387N/A MAINTENANCE_FILE = 'maintenance.txt'
3387N/A
3387N/A SQL_DUMP_FILE = 'ontohub_sql_dump.postgresql'
3387N/A REPOSITORY_FILE = 'ontohub_repositories.tar.gz'
3387N/A
3387N/A DATA_DIRS = %w(repositories git_daemon)
3387N/A
3387N/A attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
3387N/A attr_reader :dry_run, :verbose, :sql_dump_as_db_user
3387N/A
3387N/A def initialize(db_name, data_root, backup_root,
3387N/A verbose: false, dry_run: true, sql_dump_as_db_user: false)
3387N/A @db_name = db_name
3387N/A @backup_root = Pathname.new(backup_root)
3387N/A @data_root = Pathname.new(data_root)
3387N/A @data_root_basename = @data_root.basename.to_s
3387N/A @data_dirs = DATA_DIRS.map { |dir| File.join(@data_root_basename, dir) }
3387N/A
3387N/A @dry_run = dry_run
3387N/A @verbose = verbose
3387N/A @sql_dump_as_db_user = sql_dump_as_db_user
3387N/A end
3387N/A
3387N/A def create
3387N/A puts 'Creating backup...'
3387N/A enable_maintenance_mode
3387N/A initialize_backup
3387N/A create_sql_dump
3387N/A create_repository_archive
3387N/A # We needed to create the directory for the script to continue later on.
3387N/A Dir.rmdir(backup_instance_dir) if dry_run
3387N/A disable_maintenance_mode
3387N/A puts "Created backup in #{backup_instance_dir}"
3387N/A self.class.prune(backup_root)
3387N/A end
3387N/A
3387N/A def restore(backup_name)
3387N/A enable_maintenance_mode
3387N/A initialize_restore(backup_name)
3387N/A restore_sql_dump
3387N/A restore_repository_archive
3387N/A disable_maintenance_mode
3387N/A puts "Restored backup from #{backup_instance_dir}"
3387N/A end
3387N/A
3387N/A def self.prune(backup_root)
3387N/A if !Dir.exists?(backup_root)
3387N/A $stderr.puts "Nothing to prune: There is no backup directory."
3387N/A return
3387N/A end
3387N/A now = Time.now
3387N/A backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
3387N/A backup = backup_root.join(dir)
3387N/A if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
3387N/A puts "removing old backup: #{dir}"
3387N/A FileUtils.rm_r(backup)
3387N/A end
3387N/A end
3387N/A end
3387N/A
3387N/A protected
3387N/A
3387N/A def new_backup_name
3387N/A Time.now.strftime("%Y-%m-%d_%H-%M-%S")
3387N/A end
3387N/A
3387N/A def initialize_backup
3387N/A @backup_instance_dir = backup_root.join(new_backup_name)
3387N/A puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
3387N/A # Create directory even in dry run to let the script continue.
3387N/A FileUtils.mkdir_p(backup_instance_dir)
3387N/A end
3387N/A
3387N/A def create_sql_dump
3387N/A puts 'Creating SQL dump...'
3387N/A Dir.chdir(backup_instance_dir) do
3387N/A exec('pg_dump', *pg_user_switch, '-Fc', db_name, '-f', SQL_DUMP_FILE)
3387N/A end
3387N/A end
3387N/A
3387N/A def create_repository_archive
3387N/A puts 'Creating repository archive...'
3387N/A Dir.chdir(data_root.join('..')) do
3387N/A archive_file = backup_instance_dir.join(REPOSITORY_FILE)
3387N/A exec('tar', verbose ? '-v' : '', '-cf', archive_file.to_s, *@data_dirs)
3387N/A end
3387N/A end
3387N/A
3387N/A def initialize_restore(backup_name)
3387N/A @backup_instance_dir = backup_root.join(backup_name)
3387N/A unless Dir.exists?(backup_instance_dir)
3387N/A $stderr.puts (
3387N/A "Error: Backup '#{backup_name}' does not exist in #{backup_root}.")
3387N/A exit
3387N/A end
3387N/A end
3387N/A
3387N/A def restore_sql_dump
3387N/A 'Restoring SQL dump...'
3387N/A Dir.chdir(backup_instance_dir) do
3387N/A exec('pg_restore', '-n', 'public',
3387N/A '-c', *pg_user_switch,
3387N/A '-d', db_name,
3387N/A SQL_DUMP_FILE)
3387N/A end
3387N/A end
3387N/A
3387N/A def restore_repository_archive
3387N/A puts 'Restoring repository archive...'
3387N/A Dir.chdir(data_root.join('..')) do
3387N/A tmpdir = Dir.mktmpdir
3387N/A move_data_dirs_to_tmpdir(tmpdir)
3387N/A begin
3387N/A extract_archive
3387N/A remove_tmpdir(tmpdir)
3387N/A rescue => e
3387N/A puts <<-MSG
3387N/A
3387N/AAn error occured while restoring the repositories:
3387N/A#{e.message}
3387N/AYou can find the pre-restore repositories at #{tmpdir}
3387N/ADo something about it.
3387N/A MSG
3387N/A raise e
3387N/A end
3387N/A end
3387N/A end
3387N/A
3387N/A def move_data_dirs_to_tmpdir(tmpdir)
3387N/A puts "FileUtils.mv(#{@data_dirs}, #{tmpdir})" if verbose
3387N/A FileUtils.mv(@data_dirs, tmpdir) unless dry_run
3387N/A rescue Errno::EACCES
3387N/A puts <<-MSG
3387N/A
3387N/AAs the current user I have no access to move the repository data
3387N/Adirectories #{@data_dirs.join(' ')} to a temporary directory #{tmpdir}.
3387N/AThis is used as a backup for the case of an error while restoring.
3387N/ATo continue, I try the command again using sudo.
3387N/A MSG
3387N/A exec('sudo', 'mv', *@data_dirs, tmpdir)
3387N/A end
3387N/A
3387N/A def extract_archive
3387N/A archive_file = backup_instance_dir.join(REPOSITORY_FILE)
3387N/A puts <<-MSG
3387N/A
3387N/ASuper user privileges are needed to reset the file permissions as
3387N/Athey were before the backup. If you refuse to enter the password
3387N/A(Ctl-C) or enter a wrong password, only the permissions will not be
3387N/Arestored and all restored files will belong to the current user/group.
3387N/A MSG
3387N/A try_as_sudo_with_fallback('tar', verbose ? '-v' : '', '-xf',
3387N/A archive_file.to_s, *@data_dirs)
3387N/A end
3387N/A
3387N/A def remove_tmpdir(tmpdir)
3387N/A puts "FileUtils.remove_entry(#{tmpdir})" if verbose
3387N/A FileUtils.remove_entry(tmpdir) # even do this in dry run
3387N/A rescue Errno::EACCES
3387N/A puts <<-MSG
3387N/A
3387N/AAs the current user I have no access to remove the temporary
3387N/Adirectory #{tmpdir}.
3387N/ATo continue, I try the command again using sudo.
3387N/A MSG
3387N/A exec('sudo', 'rm', '-r', tmpdir)
3387N/A end
3387N/A
3387N/A def enable_maintenance_mode
3387N/A puts 'Enabling maintenance mode...'
3387N/A if File.exist?(maintenance_file)
3387N/A $stderr.puts 'Maintenance mode was already enabled.'
3387N/A $stderr.puts "Please check the file #{maintenance_file}"
3387N/A $stderr.puts 'Aborting.'
3387N/A exit
3387N/A end
3387N/A puts "FileUtils.touch #{maintenance_file}" if verbose
3387N/A FileUtils.touch maintenance_file unless dry_run
3387N/A end
3387N/A
3387N/A def disable_maintenance_mode
3387N/A puts 'Disabling maintenance mode...'
puts "FileUtils.rm #{maintenance_file}" if verbose
FileUtils.rm maintenance_file unless dry_run
end
def exec(*args)
puts "[executing next command in #{Dir.getwd}]" if verbose
out = args.join(' ')
puts out if verbose
system(*args) unless dry_run
end
def try_as_sudo_with_fallback(*args)
_out, _err, exit_code = exec('sudo', *args)
unless exit_code.success?
sudo_not_given_fallback(*args) # Wrong sudo password
end
rescue Exception => e
raise e unless e.is_a?(Interrupt) # Ctrl-C when asked for password
sudo_not_given_fallback(*args)
end
def sudo_not_given_fallback(*args)
puts 'Super user privileges not granted. Trying as normal user.'
exec(*args)
end
def maintenance_file
data_root.join(MAINTENANCE_FILE)
end
def pg_user_switch
sql_dump_as_db_user ? %w(-U ontohub) : []
end
def self.backup_dirs_allowed_to_delete(entries)
entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
end
end
end
def data_root(rails_root)
if on_development_system?(rails_root)
File.realpath(rails_root.join('data'))
else
ENV['DATA_ROOT'] ||'/data/git'
end
end
def on_development_system?(rails_root)
data_path = rails_root.join('data')
File.exist?(data_path) && !File.symlink?(data_path)
end
# Don't allow this to be run as the root user.
if ENV['USER'] == 'root'
puts 'Running this script as the root user is disabled.'
puts 'Please run it as a normal user that has sudo privileges, e.g. ontohub.'
exit
end
# We assume, this script runs in "RAILS_ROOT/script/".
RAILS_ROOT = Pathname.new(__FILE__).dirname.join('..')
BACKUP_ROOT_PRODUCTION = '/local/home/ontohub/ontohub_data_backup'
DATABASE =
if on_development_system?(RAILS_ROOT)
'ontohub_development'
else
'ontohub'
end
BACKUP_ROOT =
if on_development_system?(RAILS_ROOT)
RAILS_ROOT.join('tmp', 'backup')
else
File.realpath(BACKUP_ROOT_PRODUCTION)
end
backup = Backup::Backup.new(DATABASE, data_root(RAILS_ROOT), BACKUP_ROOT,
sql_dump_as_db_user: true,
dry_run: false, verbose: true)
case ARGV.first
when 'create'
backup.create
when 'restore'
if ARGV.length == 1
$stderr.puts(
'To restore a backup, you need to specify one with the arguments')
$stderr.puts('"restore backup_name"')
exit
end
backup_name = ARGV[1]
backup.restore(backup_name)
when 'prune'
Backup::Backup.prune(BACKUP_ROOT)
else
$stderr.puts 'unknown or missing parameter'
$stderr.puts 'use parameter "create" or "restore <backup_name>" or "prune"'
exit
end