backup revision 4fbbf628eda08ffdeb98cb41fb7e9001050eec3a
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# You can find more extensive documentation of this script at
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# https://github.com/ontohub/ontohub/blob/staging/doc/backup_and_restore_of_ontohub_data.md
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# This backup script creates and restores backups of ontohub data. It includes:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# * bare git repositories (data/repositories)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# * named symlinks to git repositories (data/git_daemon)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# * the postgres database
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# First note: Run this as the root user, e.g. with sudo.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# To create a backup, run this script with the argument `create`:
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# Then a backup named with the current date and time is created in the
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# backup directory (see below).
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# To restore a backup, run this script with the argument `restore <backup name>`
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# # script/backup restore 2015-01-01_00-00
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# Then the selected backup is fully restored
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# Backup directory
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# For development machines, the backup directory is:
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# <rails root>/tmp/backup/
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# And for production machines, the backup directory is:
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# Super user privileges
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# To create and restore, we need root privileges. Otherwise file modes are not
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen# preserved. This script will call `sudo` when needed and inform you about the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# reason for calling `sudo`. If you don't allow sudo, a backup will be created
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# or restored anyway, but the file modes and ownership are not preserved.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# Then, you need to adjust them manually.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# Maintenance mode
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# While backing up and restoring the data, the maintenance mode is activated.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen# This way we guarantee data consistency of the backup.
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainenrequire 'fileutils'
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainenrequire 'pathname'
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenrequire 'open3'
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen # Amount of backups that have to be there at least
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen # Backups are kept for at least 365 days
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen REPOSITORY_FILE = 'ontohub_repositories.tar.gz'
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen # Use 'sudo' on most systems
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen attr_reader :dry_run, :verbose, :sql_dump_as_db_user
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen def initialize(db_name, data_root, backup_root,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen verbose: false, dry_run: true, sql_dump_as_db_user: nil, user: nil)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen @data_root_basename = @data_root.basename.to_s
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen @data_dirs = DATA_DIRS.map { |dir| File.join(@data_root_basename, dir) }
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen # We needed to create the directory for the script to continue later on.
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen puts "Created backup in #{backup_instance_dir}"
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen puts "Restored backup from #{backup_instance_dir}"
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen $stderr.puts "Nothing to prune: There is no backup directory."
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen @backup_instance_dir = backup_root.join(new_backup_name)
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen # Create directory even in dry run to let the script continue.
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen exec('pg_dump', *pg_user_switch, '-Fc', db_name, '-f', SQL_DUMP_FILE,
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen archive_file = backup_instance_dir.join(REPOSITORY_FILE)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen exec('tar', verbose ? '-v' : '', '-cf', archive_file.to_s, *@data_dirs,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen @backup_instance_dir = backup_root.join(backup_name)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "Error: Backup '#{backup_name}' does not exist in #{backup_root}.")
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo SirainenAn error occured while restoring the repositories:
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenYou can find the pre-restore repositories at #{tmpdir}
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo SirainenDo something about it.
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen def move_data_dirs_to_tmpdir(tmpdir)
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen puts "FileUtils.mv(#{@data_dirs}, #{tmpdir})" if verbose
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen FileUtils.mv(@data_dirs, tmpdir) unless dry_run
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen rescue Errno::EACCES
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo SirainenAs the current user I have no access to move the repository data
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainendirectories #{@data_dirs.join(' ')} to a temporary directory #{tmpdir}.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenThis is used as a backup for the case of an error while restoring.
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo SirainenTo continue, I try the command again using sudo.
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen exec('mv', *@data_dirs, tmpdir, user: 'root')
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen def extract_archive
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen archive_file = backup_instance_dir.join(REPOSITORY_FILE)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo SirainenSuper user privileges are needed to reset the file permissions as
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenthey were before the backup. If you refuse to enter the password
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen(Ctl-C) or enter a wrong password, only the permissions will not be
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenrestored and all restored files will belong to the current user/group.
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen try_as_sudo_with_fallback('tar', verbose ? '-v' : '', '-xf',
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen archive_file.to_s, *@data_dirs)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen def remove_tmpdir(tmpdir)
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen puts "FileUtils.remove_entry(#{tmpdir})" if verbose
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen FileUtils.remove_entry(tmpdir) # even do this in dry run
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen rescue Errno::EACCES
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenAs the current user I have no access to remove the temporary
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainendirectory #{tmpdir}.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenTo continue, I try the command again using sudo.
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen exec('rm', '-r', tmpdir, user: 'root')
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen def enable_maintenance_mode
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen puts 'Enabling maintenance mode...'
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen if File.exist?(maintenance_file)
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen $stderr.puts 'Maintenance mode was already enabled.'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts "Please check the file #{maintenance_file}"
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts 'Aborting.'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen puts "FileUtils.touch #{maintenance_file}" if verbose
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen FileUtils.touch maintenance_file unless dry_run
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen def disable_maintenance_mode
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen puts 'Disabling maintenance mode...'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen puts "FileUtils.rm #{maintenance_file}" if verbose
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen FileUtils.rm maintenance_file unless dry_run
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen # Execute a command as the given user.
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen def exec(*args, user: nil)
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen puts "[executing next command in #{Dir.getwd}]" if verbose
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen out = args.join(' ')
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen puts out if verbose
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen if user == 'root'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen system([sudo, *args])
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen system([sudo, 'su', '-', user, '-c', escape_arguments(args)])
484efa22e65c509f787dbbc892351146c726c257Timo Sirainen system(*args)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen def escape_arguments(args)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen ([args[0]] + *args[1..-1].map { |a| %("#{a}")}).join(' ')
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen def maintenance_file
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen data_root.join(MAINTENANCE_FILE)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen def pg_user_switch
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen sql_dump_as_db_user ? %W(-U #{sql_dump_as_db_user}) : []
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen def self.backup_dirs_allowed_to_delete(entries)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainendef data_root(rails_root)
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen if on_development_system?(rails_root)
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen File.realpath(rails_root.join('data'))
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen ENV['DATA_ROOT'] ||'/data/git'
f0f9c8e94abac18f8acd91b9e724c4c32863723aTimo Sirainendef on_development_system?(rails_root)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen data_path = rails_root.join('data')
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen File.exist?(data_path) && !File.symlink?(data_path)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen# Don't allow this to be run as the root user.
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainenif ENV['USER'] != 'root'
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen puts 'Running this script as a normal user is disabled.'
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen puts 'Please run it as root.'
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen# We assume, this script runs in "RAILS_ROOT/script/".
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenRAILS_ROOT = Pathname.new(__FILE__).dirname.join('..')
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenBACKUP_ROOT_PRODUCTION = '/local/home/ontohub/ontohub_data_backup'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenUSER = 'ontohub'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if on_development_system?(RAILS_ROOT)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen 'ontohub_development'
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen if on_development_system?(RAILS_ROOT)
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen RAILS_ROOT.join('tmp', 'backup')
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen File.realpath(BACKUP_ROOT_PRODUCTION)
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainenbackup = Backup::Backup.new(DATABASE, data_root(RAILS_ROOT), BACKUP_ROOT,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sql_dump_as_db_user: on_development_system?(RAILS_ROOT) ? 'postgres' : 'ontohub',
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen dry_run: false, verbose: true)
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainencase ARGV.first
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen backup.create
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenwhen 'restore'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ARGV.length == 1
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts(
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen 'To restore a backup, you need to specify one with the arguments')
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts('"restore backup_name"')
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen backup_name = ARGV[1]
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen backup.restore(backup_name)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen Backup::Backup.prune(BACKUP_ROOT)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts 'unknown or missing parameter'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts 'use parameter "create" or "restore <backup_name>" or "prune"'