backup revision 2dbc668d1e44c95db1857d3968bcde7517852bea
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenrequire 'fileutils'
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenrequire 'pathname'
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenrequire 'open3'
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenrequire 'pry-byebug'
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen # Amount of backups that have to be there at least
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen # Backups are valid for 7 days
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen REPOSITORY_FILE = 'ontohub_repositories.tar.gz'
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen DATA_DIRS = %w(data/repositories data/git_daemon)
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen attr_reader :db_name, :data_root, :backup_root, :backup_instance_dir
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen attr_reader :dry_run, :verbose, :sql_dump_as_postgres_user
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen def initialize(db_name, data_root, backup_root,
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen verbose: false, dry_run: true, sql_dump_as_postgres_user: false)
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen @sql_dump_as_postgres_user = sql_dump_as_postgres_user
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen # We needed to create the directory for the script to continue later on.
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen puts "created backup in #{backup_instance_dir}"
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen puts "restored backup from #{backup_instance_dir}"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen $stderr.puts "Nothing to prune: There is no backup directory."
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen backup_dirs_allowed_to_delete(Dir.new(backup_root).entries).each do |dir|
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen if now - File.new(backup).ctime > BACKUPS_VALIDITY_TIME
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen @backup_instance_dir = backup_root.join(new_backup_name)
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen puts "FileUtils.mkdir_p #{backup_instance_dir}" if verbose
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen # Create directory even in dry run to let the script continue.
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen exec('pg_dump', pg_user_switch, '-Fc', db_name,
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen archive_file = backup_instance_dir.join(REPOSITORY_FILE)
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen exec('tar', verbose ? '-v' : '', '-czf', archive_file.to_s,
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen @backup_instance_dir = backup_root.join(backup_name)
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen $stderr.puts "Error: Backup '#{backup_name}' does not exist in #{backup_root}."
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen exec('pg_restore', '-c', pg_user_switch, '-d', db_name, SQL_DUMP_FILE)
299183fbb6ec5d0828a0880da372540421ac4665Timo SirainenAn error occured while restoring the repositories:
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenYou can find the pre-restore repositories at #{tmpdir}
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenDo something about it.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen def move_data_dirs_to_tmpdir(tmpdir)
2e937ed8585299b2e879a28314902a5f644813d2Timo Sirainen puts "FileUtils.mv(#{DATA_DIRS}, #{tmpdir})" if verbose
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen FileUtils.mv(DATA_DIRS, tmpdir) unless dry_run
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen rescue Errno::EACCES
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo SirainenAs the current user I have no access to move the repository data
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainendirectories #{DATA_DIRS.join(' ')} to a temporary directory #{tmpdir}.
4d25408732be27e91f0430f71e87242760c2517cTimo SirainenThis is used as a backup for the case of an error while restoring.
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo SirainenTo continue, I try the command again using sudo.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen exec('sudo', 'mv', *DATA_DIRS.map(&:to_s), tmpdir)
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen def extract_archive
c3412ddeb9abc13f99d3caf50faf76cd99f7e9d2Timo Sirainen archive_file = backup_instance_dir.join(REPOSITORY_FILE)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo SirainenSuper user privileges are needed to reset the file permissions as
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenthey were before the backup. If you refuse to enter the password
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen(Crtl-C) or enter a wrong password, only the permissions will not be
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenrestored and all restored files will belong to the current user/group.
1e76a5b92f9d82d557f81f080f3dfad1c9d8f200Timo Sirainen try_as_sudo_with_fallback('tar', verbose ? '-v' : '', '-xzf',
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen archive_file.to_s, *DATA_DIRS.map(&:to_s))
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen def remove_tmpdir(tmpdir)
e8a35266a5ceacdfafeeffd6bddae77931ff97ebTimo Sirainen puts "FileUtils.remove_entry(#{tmpdir})" if verbose
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen FileUtils.remove_entry(tmpdir) # even do this in dry run
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen rescue Errno::EACCES
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo SirainenAs the current user I have no access to remove the temporary
b08b33d1f5ce3721dc2d83586c9cb0ca141331fdTimo Sirainendirectory #{tmpdir}.
b08b33d1f5ce3721dc2d83586c9cb0ca141331fdTimo SirainenTo continue, I try the command again using sudo.
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen exec('sudo', 'rm', '-r', tmpdir)
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen def enable_maintenance_mode
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen puts "FileUtils.touch #{maintenance_file}" if verbose
b08b33d1f5ce3721dc2d83586c9cb0ca141331fdTimo Sirainen FileUtils.touch maintenance_file unless dry_run
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen def disable_maintenance_mode
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen puts "FileUtils.rm #{maintenance_file}" if verbose
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen FileUtils.rm maintenance_file unless dry_run
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen def exec(*args, file_dest: nil)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen puts "[executing next command in #{Dir.getwd}]" if verbose
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen out = args.join(' ')
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen out << " > #{file_dest}" if file_dest
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen puts out if verbose
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen Subprocess.run(*args, file_dest: file_dest) unless dry_run
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen def try_as_sudo_with_fallback(*args)
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen unless exec('sudo', *args)
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen sudo_not_given_fallback(*args) # Wrong sudo password
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen rescue Exception => e
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen raise e unless e.is_a?(Interrupt) # Ctrl-C when asked for password
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen sudo_not_given_fallback(*args)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen def sudo_not_given_fallback(*args)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen puts 'Super user privileges not granted. Trying as normal user.'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen def maintenance_file
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen data_root.join(MAINTENANCE_FILE)
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen def pg_user_switch
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen sql_dump_as_postgres_user ? ' -U postgres' : ''
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen def self.backup_dirs_allowed_to_delete(entries)
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen entries.reject{ |entry| %w(. ..).include?(entry) }[0..-(BACKUPS_COUNT+1)]
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen class Subprocess
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen def self.run(*args, file_dest: nil)
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen stdin, stdout, stderr, wait_thr, io_dest = run_streaming(*args,
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen file_dest: file_dest)
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen exit_code = wait_thr.value # wait for the process to finish
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen io_dest.close if io_dest
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen [stdout.read, stderr.read, exit_code]
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen def self.run_streaming(*args, file_dest: nil)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen stdin, stdout, stderr, wait_thr = Open3.popen3(*args)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen io_dest = nil
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen io_dest = File.open(file_dest, 'w')
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen IO.copy_stream(stdout, io_dest)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen [stdin, stdout, stderr, wait_thr, io_dest]
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainendef data_root(rails_root)
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen File.realpath(rails_root.join('data'))
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainendef on_development_system?(rails_root)
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen !File.symlink?(rails_root.join('data'))
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen# We assume, this script runs in "RAILS_ROOT/script/".
73b50eecfc31750a312e2f940023f522eb07178cTimo SirainenRAILS_ROOT = Pathname.new(__FILE__).dirname.join('..')
73b50eecfc31750a312e2f940023f522eb07178cTimo SirainenDATABASE = if on_development_system?(RAILS_ROOT)
f0f9c8e94abac18f8acd91b9e724c4c32863723aTimo Sirainen 'ontohub_development'
7f3be7d885c75cdd77f536929a45bc9764595960Timo SirainenBACKUP_ROOT = if on_development_system?(RAILS_ROOT)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen RAILS_ROOT.join('tmp', 'backup')
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen File.realpath('/home/ontohub/ontohub_data_backup')
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainenbackup = Backup::Backup.new(DATABASE, data_root(RAILS_ROOT), BACKUP_ROOT,
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen sql_dump_as_postgres_user: on_development_system?(RAILS_ROOT),
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen dry_run: false, verbose: true)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainencase ARGV.first
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen backup.create
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainenwhen 'restore'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ARGV.length == 1
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts 'To restore a backup, you need to specify one with the arguments'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts '"restore backup_name"'
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen backup_name = ARGV[1]
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen backup.restore(backup_name)
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen Backup::Backup.prune(BACKUP_ROOT)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts 'unknown or missing parameter'
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen $stderr.puts 'use parameter "create" or "restore <backup_name>" or "prune"'