backup revision 706a446ea5c5c1644b53a82fcadd8c9080ba3b4f
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa# The first line is for deployment machines only. For local machines, use:
abc834c020080fe44a1ea4e34278327e99e3e12eEugen Kuksa# You can find more extensive documentation of this script at
abc834c020080fe44a1ea4e34278327e99e3e12eEugen Kuksa# https://github.com/ontohub/ontohub/blob/staging/doc/backup_and_restore_of_ontohub_data.md
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# Description
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# This backup script creates and restores backups of ontohub data. It includes:
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# * bare git repositories (data/repositories)
6bbb71b52f053c148669e5b128e236781150da1fEugen Kuksa# * named symlinks to git repositories (data/git_daemon and data/git_ssh)
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# * the postgres database
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa# First note: Run this as the root user, e.g. with sudo.
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# To create a backup, run this script with the argument `create`:
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# Then a backup named with the current date and time is created in the
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# backup directory (see below).
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# To restore a backup, run this script with the argument `restore <backup name>`
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa# # script/backup restore 2015-01-01_00-00
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# Then the selected backup is fully restored
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# Backup directory
8230471f68bf2a663d94872531d0bcf16d426bbeEugen Kuksa# For production machines, the backup directory is:
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# Super user privileges
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# To create and restore, we need root privileges. Otherwise file modes are not
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# preserved. This script will call `sudo` when needed and inform you about the
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# reason for calling `sudo`. If you don't allow sudo, a backup will be created
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# or restored anyway, but the file modes and ownership are not preserved.
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# Then, you need to adjust them manually.
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# Maintenance mode
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# While backing up and restoring the data, the maintenance mode is activated.
f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa# This way we guarantee data consistency of the backup.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksarequire 'fileutils'
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksarequire 'pathname'
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksarequire 'open3'
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa # Amount of backups that have to be there at least
d89f470f7da0b9f8295d0ac0defff09884894b8bEugen Kuksa # Backups are kept for at least 365 days
4fbbf628eda08ffdeb98cb41fb7e9001050eec3aEugen Kuksa # Use 'sudo' on most systems
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
67daeb8066e4460cb820db60c45138dd48309bb9Eugen Kuksa attr_reader :dry_run, :verbose, :sql_dump_as_db_user
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa verbose: false, dry_run: true, sql_dump_as_db_user: nil,
a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa @data_dirs = DATA_DIRS.map { |dir| File.join(@data_root_basename, dir) }
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa # We needed to create the directory for the script to continue later on.
4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa puts "Restored backup from #{backup_instance_dir}"
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa $stderr.puts "Nothing to prune: There is no backup directory."
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa @backup_instance_dir = backup_root.join(new_backup_name)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa # Create directory even in dry run to let the script continue.
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa puts "FileUtils.chown #{@user} #{@group} #{backup_instance_dir}" if verbose
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa FileUtils.chown(@user, @group, backup_instance_dir)
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa exec('pg_dump', *pg_user_switch, '-Fc', db_name,
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa '-f', backup_instance_dir.join(SQL_DUMP_FILE), user: @user)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa archive_file = backup_instance_dir.join(REPOSITORY_FILE)
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa exec('tar', verbose ? '-v' : '', '-cf', archive_file.to_s, *@data_dirs,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa @backup_instance_dir = backup_root.join(backup_name)
4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa "Error: Backup '#{backup_name}' does not exist in #{backup_root}.")
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaAn error occured while restoring the repositories:
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaYou can find the pre-restore repositories at #{tmpdir}
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaDo something about it.
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa def move_data_dirs_to_tmpdir(tmpdir)
a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa puts "FileUtils.mv(#{@data_dirs}, #{tmpdir})" if verbose
a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa FileUtils.mv(@data_dirs, tmpdir) unless dry_run
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa rescue Errno::EACCES
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaAs the current user I have no access to move the repository data
a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksadirectories #{@data_dirs.join(' ')} to a temporary directory #{tmpdir}.
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaThis is used as a backup for the case of an error while restoring.
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaTo continue, I try the command again using sudo.
108ccf954b02c757f7d267e5eb9260f71ff39862Eugen Kuksa exec('mv', *@data_dirs, tmpdir, user: 'root') unless dry_run
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa def extract_archive
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa archive_file = backup_instance_dir.join(REPOSITORY_FILE)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaSuper user privileges are needed to reset the file permissions as
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksathey were before the backup. If you refuse to enter the password
6cdc461aada609d57d50ff675d29b15378717ff2Eugen Kuksa(Ctl-C) or enter a wrong password, only the permissions will not be
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksarestored and all restored files will belong to the current user/group.
a9b1a876554e3539bd8e70e6081888c8a0134f22Eugen Kuksa exec('tar', verbose ? 'vxf' : 'xf', archive_file.to_s, *@data_dirs,
a9b1a876554e3539bd8e70e6081888c8a0134f22Eugen Kuksa user: 'root')
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa def remove_tmpdir(tmpdir)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa puts "FileUtils.remove_entry(#{tmpdir})" if verbose
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa FileUtils.remove_entry(tmpdir) # even do this in dry run
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa rescue Errno::EACCES
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaAs the current user I have no access to remove the temporary
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksadirectory #{tmpdir}.
b51057b860560bf3ee454c03a121af3d5d34f482Eugen KuksaTo continue, I try the command again using sudo.
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa exec('rm', '-r', tmpdir, user: 'root')
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa def enable_maintenance_mode
4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa puts 'Enabling maintenance mode...'
bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa if File.exist?(maintenance_file)
bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa $stderr.puts 'Maintenance mode was already enabled.'
bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa $stderr.puts "Please check the file #{maintenance_file}"
bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa $stderr.puts 'Aborting.'
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa puts "FileUtils.touch #{maintenance_file}" if verbose
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa FileUtils.touch maintenance_file unless dry_run
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa def disable_maintenance_mode
4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa puts 'Disabling maintenance mode...'
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa puts "FileUtils.rm #{maintenance_file}" if verbose
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa FileUtils.rm maintenance_file unless dry_run
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa # Execute a command as the given user.
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa def exec(*args, user: nil)
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa print "[executing next command in #{Dir.getwd}" if verbose
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa print " as user #{user}" if verbose && user
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa puts "]" if verbose
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa out = args.join(' ')
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa puts out if verbose
4fbbf628eda08ffdeb98cb41fb7e9001050eec3aEugen Kuksa if user == 'root'
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa exec_system(*[sudo, *args])
ab0efd0754ad1487560385983a635086fdd09084Eugen Kuksa # This looks strange because of the combination of + and sudo.
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa # It is needed on our deployment machines to get the environment right.
ab0efd0754ad1487560385983a635086fdd09084Eugen Kuksa # On other machines, remove the call of +.
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa exec_system(*['+', 'sudo', '-u', user, 'bash', '-c',
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa "cd #{Dir.getwd} && #{escape_arguments(args)}"])
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa exec_system(*args)
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa def exec_system(*args)
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa # puts args.join(' ') # For debugging
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa system(*args)
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa def escape_arguments(args)
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa rest = args[1..-1].map do |arg|
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa if arg.to_s.include?(' ')
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa %("#{arg.gsub('"', '\"')}")
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa ([args[0]] + rest).join(' ')
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa def maintenance_file
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa data_root.join(MAINTENANCE_FILE)
6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa def pg_user_switch
57c81a32dff6182f040b4f852892144641a62fe5Eugen Kuksa sql_dump_as_db_user ? %W(-U #{sql_dump_as_db_user}) : []
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa def self.backup_dirs_allowed_to_delete(entries)
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksadef data_root(rails_root)
b84d89b1aa3faa108105fc68925e33dc7eaf9d01Eugen Kuksa ENV['DATA_ROOT'] ||'/data/git'
0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksa# Don't allow this to be run as the root user.
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksaif ENV['USER'] != 'root'
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa puts 'Running this script as a normal user is disabled.'
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen Kuksa puts 'Please run it as root.'
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa# We assume, this script runs in "RAILS_ROOT/script/".
daf3e28fff47a65b53d6fb65155301763b9f166eEugen KuksaRAILS_ROOT = Pathname.new(__FILE__).dirname.join('..')
8230471f68bf2a663d94872531d0bcf16d426bbeEugen KuksaBACKUP_ROOT_PRODUCTION = '/data/ontohub_data_backup'
255e2c2477c832c7a785ae0ae77fd25add1e58e6Eugen KuksaUSER = 'ontohub'
7e19fab3af21714e410863295a27b859111ba40dEugen KuksaGROUP = 'webservd'
b84d89b1aa3faa108105fc68925e33dc7eaf9d01Eugen KuksaDATABASE = 'ontohub'
3edb9eac22c71a937953dd2ad4911d12fb22c6c6Eugen Kuksaunless File.exist?(BACKUP_ROOT_PRODUCTION)
3edb9eac22c71a937953dd2ad4911d12fb22c6c6Eugen Kuksa FileUtils.mkdir_p(BACKUP_ROOT_PRODUCTION)
b84d89b1aa3faa108105fc68925e33dc7eaf9d01Eugen KuksaBACKUP_ROOT = File.realpath(BACKUP_ROOT_PRODUCTION)
706a446ea5c5c1644b53a82fcadd8c9080ba3b4fEugen Kuksaunless File.directory?(BACKUP_ROOT)
706a446ea5c5c1644b53a82fcadd8c9080ba3b4fEugen Kuksa $stderr.puts "Target path is not a directory: #{BACKUP_ROOT}"
b1fe9054ad7c7192fe4c474363247dad15963e99Eugen Kuksabackup = Backup::Backup.new(DATABASE, data_root(RAILS_ROOT), BACKUP_ROOT,
b84d89b1aa3faa108105fc68925e33dc7eaf9d01Eugen Kuksa sql_dump_as_db_user: 'ontohub',
7e19fab3af21714e410863295a27b859111ba40dEugen Kuksa user: USER, group: GROUP,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa dry_run: false, verbose: true)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksacase ARGV.first
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksawhen 'create'
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa backup.create
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksawhen 'restore'
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if ARGV.length == 1
4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa $stderr.puts(
4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa 'To restore a backup, you need to specify one with the arguments')
4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa $stderr.puts('"restore backup_name"')
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa backup_name = ARGV[1]
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa backup.restore(backup_name)
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa Backup::Backup.prune(BACKUP_ROOT)
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa $stderr.puts 'unknown or missing parameter'
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa $stderr.puts 'use parameter "create" or "restore <backup_name>" or "prune"'