backup revision a8028fd2789e323040de08827a0fe1f7d36fde2b
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa# You can find more extensive documentation of this script at
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# https://github.com/ontohub/ontohub/blob/staging/doc/backup_and_restore_of_ontohub_data.md
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa# Description
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa# This backup script creates and restores backups of ontohub data. It includes:
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa# * bare git repositories (data/repositories)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# * named symlinks to git repositories (data/git_daemon)
b1fe9054ad7c7192fe4c474363247dad15963e99Eugen Kuksa# * the postgres database
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa# First note: Run this as the ontohub user, *not* as root.
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa# To create a backup, run this script with the argument `create`:
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa# Then a backup named with the current date and time is created in the
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# backup directory (see below).
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# To restore a backup, run this script with the argument `restore <backup name>`
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# $ script/backup restore 2015-01-01_00-00
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# Then the selected backup is fully restored
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# Backup directory
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# For development machines, the backup directory is:
366ce8d807067a97613cb23d49105d8a093c5015Eugen Kuksa# <rails root>/tmp/backup/
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# And for production machines, the backup directory is:
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# Super user privileges
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# To create and restore, we need root privileges. Otherwise file modes are not
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# preserved. This script will call `sudo` when needed and inform you about the
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# reason for calling `sudo`. If you don't allow sudo, a backup will be created
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# or restored anyway, but the file modes and ownership are not preserved.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# Then, you need to adjust them manually.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# Maintenance mode
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# While backing up and restoring the data, the maintenance mode is activated.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# This way we guarantee data consistency of the backup.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksarequire 'fileutils'
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksarequire 'pathname'
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksarequire 'open3'
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa # Amount of backups that have to be there at least
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa # Backups are kept for at least 365 days
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa attr_reader :dry_run, :verbose, :sql_dump_as_postgres_user
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa verbose: false, dry_run: true, sql_dump_as_postgres_user: false)
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa @data_dirs = DATA_DIRS.map { |dir| File.join(@data_root_basename, dir) }
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa @sql_dump_as_postgres_user = sql_dump_as_postgres_user
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa # We needed to create the directory for the script to continue later on.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa puts "Restored backup from #{backup_instance_dir}"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa $stderr.puts "Nothing to prune: There is no backup directory."
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa @backup_instance_dir = backup_root.join(new_backup_name)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa # Create directory even in dry run to let the script continue.
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa exec('pg_dump', *pg_user_switch, '-Fc', db_name, '-f', SQL_DUMP_FILE)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa archive_file = backup_instance_dir.join(REPOSITORY_FILE)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa exec('tar', verbose ? '-v' : '', '-cf', archive_file.to_s, *@data_dirs)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa @backup_instance_dir = backup_root.join(backup_name)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa "Error: Backup '#{backup_name}' does not exist in #{backup_root}.")
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa exec('pg_restore', '-c', *pg_user_switch, '-d', db_name, SQL_DUMP_FILE)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaAn error occured while restoring the repositories:
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaYou can find the pre-restore repositories at #{tmpdir}
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaDo something about it.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa def move_data_dirs_to_tmpdir(tmpdir)
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa puts "FileUtils.mv(#{@data_dirs}, #{tmpdir})" if verbose
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa FileUtils.mv(@data_dirs, tmpdir) unless dry_run
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa rescue Errno::EACCES
daf3e28fff47a65b53d6fb65155301763b9f166eEugen KuksaAs the current user I have no access to move the repository data
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksadirectories #{@data_dirs.join(' ')} to a temporary directory #{tmpdir}.
28001d576e67ba46ed481c5695f1e0827ff26007Eugen KuksaThis is used as a backup for the case of an error while restoring.
28001d576e67ba46ed481c5695f1e0827ff26007Eugen KuksaTo continue, I try the command again using sudo.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa exec('sudo', 'mv', *@data_dirs, tmpdir)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa def extract_archive
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa archive_file = backup_instance_dir.join(REPOSITORY_FILE)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen KuksaSuper user privileges are needed to reset the file permissions as
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksathey were before the backup. If you refuse to enter the password
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa(Ctl-C) or enter a wrong password, only the permissions will not be
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksarestored and all restored files will belong to the current user/group.
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa try_as_sudo_with_fallback('tar', verbose ? '-v' : '', '-xf',
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa archive_file.to_s, *@data_dirs)
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa def remove_tmpdir(tmpdir)
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa puts "FileUtils.remove_entry(#{tmpdir})" if verbose
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa FileUtils.remove_entry(tmpdir) # even do this in dry run
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa rescue Errno::EACCES
6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen KuksaAs the current user I have no access to remove the temporary
6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksadirectory #{tmpdir}.
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen KuksaTo continue, I try the command again using sudo.
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa exec('sudo', 'rm', '-r', tmpdir)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa def enable_maintenance_mode
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa puts 'Enabling maintenance mode...'
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa if File.exist?(maintenance_file)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa $stderr.puts 'Maintenance mode was already enabled.'
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa $stderr.puts "Please check the file #{maintenance_file}"
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa $stderr.puts 'Aborting.'
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa puts "FileUtils.touch #{maintenance_file}" if verbose
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa FileUtils.touch maintenance_file unless dry_run
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa def disable_maintenance_mode
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa puts 'Disabling maintenance mode...'
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa puts "FileUtils.rm #{maintenance_file}" if verbose
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa FileUtils.rm maintenance_file unless dry_run
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa def exec(*args)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa puts "[executing next command in #{Dir.getwd}]" if verbose
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa out = args.join(' ')
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa puts out if verbose
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa system(*args) unless dry_run
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa def try_as_sudo_with_fallback(*args)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa _out, _err, exit_code = exec('sudo', *args)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unless exit_code.success?
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa sudo_not_given_fallback(*args) # Wrong sudo password
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa rescue Exception => e
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa raise e unless e.is_a?(Interrupt) # Ctrl-C when asked for password
0b99be5ced371cca00283694e1bd53a8ac0d7b5dEugen Kuksa sudo_not_given_fallback(*args)
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa def sudo_not_given_fallback(*args)
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa puts 'Super user privileges not granted. Trying as normal user.'
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa def maintenance_file
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa data_root.join(MAINTENANCE_FILE)
b1fe9054ad7c7192fe4c474363247dad15963e99Eugen Kuksa def pg_user_switch
366ce8d807067a97613cb23d49105d8a093c5015Eugen Kuksa sql_dump_as_postgres_user ? %w(-U postgres) : []
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa def self.backup_dirs_allowed_to_delete(entries)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksadef data_root(rails_root)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if on_development_system?(rails_root)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa File.realpath(rails_root.join('data'))
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa ENV['DATA_ROOT'] ||'/data/git'
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksadef on_development_system?(rails_root)
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa data_path = rails_root.join('data')
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa File.exist?(data_path) && !File.symlink?(data_path)