backup revision 57c81a32dff6182f040b4f852892144641a62fe5
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# You can find more extensive documentation of this script at
76c42915b4b54a486d68a9020d7fceaed42f7c1eAndreas Gustafsson# https://github.com/ontohub/ontohub/blob/staging/doc/backup_and_restore_of_ontohub_data.md
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# This backup script creates and restores backups of ontohub data. It includes:
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# * bare git repositories (data/repositories)
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# * named symlinks to git repositories (data/git_daemon)
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# * the postgres database
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson# First note: Run this as the ontohub user, *not* as root.
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# To create a backup, run this script with the argument `create`:
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson# Then a backup named with the current date and time is created in the
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# backup directory (see below).
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# To restore a backup, run this script with the argument `restore <backup name>`
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson# $ script/backup restore 2015-01-01_00-00
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# Then the selected backup is fully restored
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# Backup directory
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson# For development machines, the backup directory is:
dbeb32261081835bb9ba44db68df5dfed0fda411Andreas Gustafsson# <rails root>/tmp/backup/
717e2cf05b12506b40eb03f42ea963c30c7e9f97Brian Wellington# And for production machines, the backup directory is:
40f53fa8d9c6a4fc38c0014495e7a42b08f52481David Lawrence# Super user privileges
a1747570262ed336c213aaf6bd31bc91993a46deAndreas Gustafsson# To create and restore, we need root privileges. Otherwise file modes are not
40f53fa8d9c6a4fc38c0014495e7a42b08f52481David Lawrence# preserved. This script will call `sudo` when needed and inform you about the
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson# reason for calling `sudo`. If you don't allow sudo, a backup will be created
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# or restored anyway, but the file modes and ownership are not preserved.
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson# Then, you need to adjust them manually.
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# Maintenance mode
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# While backing up and restoring the data, the maintenance mode is activated.
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson# This way we guarantee data consistency of the backup.
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafssonrequire 'fileutils'
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafssonrequire 'pathname'
2e711b60024d1ba41e7bc8010949e91aa404b244Andreas Gustafssonrequire 'open3'
e0c769c3ecb8b251fac07ffeeef8a82c1689a949Mark Andrews # Amount of backups that have to be there at least
1bc9afc3313249d656abae2d298c8d84308891d0Andreas Gustafsson # Backups are kept for at least 365 days
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson BACKUPS_VALIDITY_TIME = 365 * 60 * 60 * 24
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson SQL_DUMP_FILE = 'ontohub_sql_dump.postgresql'
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson REPOSITORY_FILE = 'ontohub_repositories.tar.gz'
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson attr_reader :dry_run, :verbose, :sql_dump_as_db_user
876a69ba28028df41d8783369338f29501a917d3Andreas Gustafsson def initialize(db_name, data_root, backup_root,
10e6498d6d7b2cfd8d822788d817fc9a3e0b0c3aDavid Lawrence verbose: false, dry_run: true, sql_dump_as_db_user: nil)
e21d199dca95aff5d50f133d6b064309e209af00Brian Wellington @data_root_basename = @data_root.basename.to_s
ef45c94e927e97ad0c804780a1eca59240088f60Andreas Gustafsson @data_dirs = DATA_DIRS.map { |dir| File.join(@data_root_basename, dir) }
51b951ab2a5e45e6a3994d033fec9b68e1f07985Mark Andrews # We needed to create the directory for the script to continue later on.
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson Dir.rmdir(backup_instance_dir) if dry_run
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson puts "Created backup in #{backup_instance_dir}"
54f6b2cfa87871782549a859ed9fc275b4b620bcAndreas Gustafsson puts "Restored backup from #{backup_instance_dir}"
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson $stderr.puts "Nothing to prune: There is no backup directory."
2cc3f128610eb9e42d7c386160665583b63882bfAndreas Gustafsson backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson @backup_instance_dir = backup_root.join(new_backup_name)
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson # Create directory even in dry run to let the script continue.
4034406393893f4d07ab07b56da3541155493855Andreas Gustafsson exec('pg_dump', *pg_user_switch, '-Fc', db_name, '-f', SQL_DUMP_FILE)
e01ecff4b1562a24e6de7e9396c60e9dffdb78ceAndreas Gustafsson archive_file = backup_instance_dir.join(REPOSITORY_FILE)
2bc0dee981fd5d9c7d7d6fe67278dfafbe614bc3Andreas Gustafsson exec('tar', verbose ? '-v' : '', '-cf', archive_file.to_s, *@data_dirs)
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson @backup_instance_dir = backup_root.join(backup_name)
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson "Error: Backup '#{backup_name}' does not exist in #{backup_root}.")
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas GustafssonAn error occured while restoring the repositories:
4334d2e8a16c4e154e8eb6fb7c7c8e8862771c8cBrian WellingtonYou can find the pre-restore repositories at #{tmpdir}
4334d2e8a16c4e154e8eb6fb7c7c8e8862771c8cBrian WellingtonDo something about it.
cbe5f076ba5595c3d63daa223ea373bef55561b2Andreas Gustafsson def move_data_dirs_to_tmpdir(tmpdir)
cbe5f076ba5595c3d63daa223ea373bef55561b2Andreas Gustafsson puts "FileUtils.mv(#{@data_dirs}, #{tmpdir})" if verbose
e0c769c3ecb8b251fac07ffeeef8a82c1689a949Mark Andrews FileUtils.mv(@data_dirs, tmpdir) unless dry_run
50baab1389a4aa811ecd6e363e310904485d969fAndreas Gustafsson rescue Errno::EACCES
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas GustafssonAs the current user I have no access to move the repository data
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafssondirectories #{@data_dirs.join(' ')} to a temporary directory #{tmpdir}.
717e2cf05b12506b40eb03f42ea963c30c7e9f97Brian WellingtonThis is used as a backup for the case of an error while restoring.
717e2cf05b12506b40eb03f42ea963c30c7e9f97Brian WellingtonTo continue, I try the command again using sudo.
a057b6e5e2d8a890184854728b26f4c86a9bdcb3Andreas Gustafsson exec('sudo', 'mv', *@data_dirs, tmpdir)
a057b6e5e2d8a890184854728b26f4c86a9bdcb3Andreas Gustafsson def extract_archive
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson archive_file = backup_instance_dir.join(REPOSITORY_FILE)
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas GustafssonSuper user privileges are needed to reset the file permissions as
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafssonthey were before the backup. If you refuse to enter the password
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafsson(Ctl-C) or enter a wrong password, only the permissions will not be
4334d2e8a16c4e154e8eb6fb7c7c8e8862771c8cBrian Wellingtonrestored and all restored files will belong to the current user/group.
876a69ba28028df41d8783369338f29501a917d3Andreas Gustafsson try_as_sudo_with_fallback('tar', verbose ? '-v' : '', '-xf',
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafsson archive_file.to_s, *@data_dirs)
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafsson def remove_tmpdir(tmpdir)
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafsson puts "FileUtils.remove_entry(#{tmpdir})" if verbose
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafsson FileUtils.remove_entry(tmpdir) # even do this in dry run
e0c769c3ecb8b251fac07ffeeef8a82c1689a949Mark Andrews rescue Errno::EACCES
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas GustafssonAs the current user I have no access to remove the temporary
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafssondirectory #{tmpdir}.
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas GustafssonTo continue, I try the command again using sudo.
a057b6e5e2d8a890184854728b26f4c86a9bdcb3Andreas Gustafsson exec('sudo', 'rm', '-r', tmpdir)
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafsson def enable_maintenance_mode
335a4599898f181f18b618a5ac4fe4e083ecd308Andreas Gustafsson puts 'Enabling maintenance mode...'
40f53fa8d9c6a4fc38c0014495e7a42b08f52481David Lawrence if File.exist?(maintenance_file)
4334d2e8a16c4e154e8eb6fb7c7c8e8862771c8cBrian Wellington $stderr.puts 'Maintenance mode was already enabled.'
4334d2e8a16c4e154e8eb6fb7c7c8e8862771c8cBrian Wellington $stderr.puts "Please check the file #{maintenance_file}"
4334d2e8a16c4e154e8eb6fb7c7c8e8862771c8cBrian Wellington $stderr.puts 'Aborting.'
40f53fa8d9c6a4fc38c0014495e7a42b08f52481David Lawrence puts "FileUtils.touch #{maintenance_file}" if verbose
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson FileUtils.touch maintenance_file unless dry_run
250ed9e230b3903b1b264dd1ed2f691fc7cd2f8fAndreas Gustafsson def disable_maintenance_mode
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson puts 'Disabling maintenance mode...'
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson puts "FileUtils.rm #{maintenance_file}" if verbose
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson FileUtils.rm maintenance_file unless dry_run
1bc9afc3313249d656abae2d298c8d84308891d0Andreas Gustafsson def exec(*args)
717e2cf05b12506b40eb03f42ea963c30c7e9f97Brian Wellington puts "[executing next command in #{Dir.getwd}]" if verbose
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson out = args.join(' ')
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson puts out if verbose
fea398993b583058fb8167902eed3eedd26f464cAndreas Gustafsson system(*args) unless dry_run
876a69ba28028df41d8783369338f29501a917d3Andreas Gustafsson def try_as_sudo_with_fallback(*args)
10e6498d6d7b2cfd8d822788d817fc9a3e0b0c3aDavid Lawrence unless exec('sudo', *args)
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson sudo_not_given_fallback(*args) # Wrong sudo password
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson rescue Exception => e
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson raise e unless e.is_a?(Interrupt) # Ctrl-C when asked for password
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson sudo_not_given_fallback(*args)
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson def sudo_not_given_fallback(*args)
7cd4c3ddd1baf5f2b204562fdba3da37c716cc78Andreas Gustafsson puts 'Super user privileges not granted. Trying as normal user.'
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson def maintenance_file
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson data_root.join(MAINTENANCE_FILE)
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson def pg_user_switch
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson sql_dump_as_db_user ? %W(-U #{sql_dump_as_db_user}) : []
113251976d99be74da788bdb78300957b77a1381Andreas Gustafsson def self.backup_dirs_allowed_to_delete(entries)
98c7e0d8ba881f06f56716d6f7098d54643f4f2fAndreas Gustafsson entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
a057b6e5e2d8a890184854728b26f4c86a9bdcb3Andreas Gustafssondef data_root(rails_root)
a057b6e5e2d8a890184854728b26f4c86a9bdcb3Andreas Gustafsson if on_development_system?(rails_root)
a057b6e5e2d8a890184854728b26f4c86a9bdcb3Andreas Gustafsson File.realpath(rails_root.join('data'))