src/import/import-tar.c

	import-tar.c revision 8c9cfc28448bf671807e7d96b580221edebd71da
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering/***
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  This file is part of systemd.
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  Copyright 2015 Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  systemd is free software; you can redistribute it and/or modify it
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  under the terms of the GNU Lesser General Public License as published by
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  the Free Software Foundation; either version 2.1 of the License, or
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  (at your option) any later version.
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  systemd is distributed in the hope that it will be useful, but
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  WITHOUT ANY WARRANTY; without even the implied warranty of
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  Lesser General Public License for more details.
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  You should have received a copy of the GNU Lesser General Public License
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering  along with systemd; If not, see <http://www.gnu.org/licenses/>.
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering***/
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include <linux/fs.h>
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "sd-daemon.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "sd-event.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "util.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "path-util.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "btrfs-util.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "hostname-util.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "copy.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "mkdir.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "rm-rf.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "ratelimit.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "machine-pool.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "qcow2-util.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "import-compress.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "import-common.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "import-tar.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering#include "process-util.h"
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poetteringstruct TarImport {
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        sd_event *event;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        char *image_root;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        TarImportFinished on_finished;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        void *userdata;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        char *local;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        bool force_local;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        bool read_only;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        bool grow_machine_directory;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        char *temp_path;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        char *final_path;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        int input_fd;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        int tar_fd;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        ImportCompress compress;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        uint64_t written_since_last_grow;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        sd_event_source *input_event_source;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        uint8_t buffer[16*1024];
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        size_t buffer_size;
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        uint64_t written_compressed;
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        uint64_t written_uncompressed;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        struct stat st;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        pid_t tar_pid;
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        unsigned last_percent;
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        RateLimit progress_rate_limit;
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek};
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-SzmekTarImport* tar_import_unref(TarImport *i) {
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        if (!i)
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                return NULL;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        sd_event_source_unref(i->input_event_source);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        if (i->tar_pid > 1) {
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek                (void) kill_and_sigcont(i->tar_pid, SIGKILL);
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek                (void) wait_for_terminate(i->tar_pid, NULL);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        }
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        if (i->temp_path) {
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                free(i->temp_path);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        }
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        import_compress_free(&i->compress);
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        sd_event_unref(i->event);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        safe_close(i->tar_fd);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        free(i->final_path);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        free(i->image_root);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        free(i->local);
f23e83b156da8966e43e303dced5439503450d21Zbigniew Jędrzejewski-Szmek        free(i);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        return NULL;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering}
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poetteringint tar_import_new(
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                TarImport **ret,
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                sd_event *event,
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                const char *image_root,
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                TarImportFinished on_finished,
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                void *userdata) {
2c86ba5a51b29e41bdfde6771d93cc5317b253cfZbigniew Jędrzejewski-Szmek
2c86ba5a51b29e41bdfde6771d93cc5317b253cfZbigniew Jędrzejewski-Szmek        _cleanup_(tar_import_unrefp) TarImport *i = NULL;
2c86ba5a51b29e41bdfde6771d93cc5317b253cfZbigniew Jędrzejewski-Szmek        int r;
2c86ba5a51b29e41bdfde6771d93cc5317b253cfZbigniew Jędrzejewski-Szmek
2c86ba5a51b29e41bdfde6771d93cc5317b253cfZbigniew Jędrzejewski-Szmek        assert(ret);
2c86ba5a51b29e41bdfde6771d93cc5317b253cfZbigniew Jędrzejewski-Szmek
2c86ba5a51b29e41bdfde6771d93cc5317b253cfZbigniew Jędrzejewski-Szmek        i = new0(TarImport, 1);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        if (!i)
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                return -ENOMEM;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        i->input_fd = i->tar_fd = -1;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        i->on_finished = on_finished;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        i->userdata = userdata;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        RATELIMIT_INIT(i->progress_rate_limit, 100 * USEC_PER_MSEC, 1);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        i->last_percent = (unsigned) -1;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        i->image_root = strdup(image_root ?: "/var/lib/machines");
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        if (!i->image_root)
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                return -ENOMEM;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        if (event)
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                i->event = sd_event_ref(event);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        else {
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                r = sd_event_default(&i->event);
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                if (r < 0)
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering                        return r;
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering        }
dc83f27a7cf03757dec11a69ec18504ad4ea8f89Lennart Poettering
        *ret = i;
        i = NULL;

        return 0;
}

static void tar_import_report_progress(TarImport *i) {
        unsigned percent;
        assert(i);

        /* We have no size information, unless the source is a regular file */
        if (!S_ISREG(i->st.st_mode))
                return;

        if (i->written_compressed >= (uint64_t) i->st.st_size)
                percent = 100;
        else
                percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size);

        if (percent == i->last_percent)
                return;

        if (!ratelimit_test(&i->progress_rate_limit))
                return;

        sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
        log_info("Imported %u%%.", percent);

        i->last_percent = percent;
}

static int tar_import_finish(TarImport *i) {
        int r;

        assert(i);
        assert(i->tar_fd >= 0);
        assert(i->temp_path);
        assert(i->final_path);

        i->tar_fd = safe_close(i->tar_fd);

        if (i->tar_pid > 0) {
                r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
                i->tar_pid = 0;
                if (r < 0)
                        return r;
        }

        if (i->read_only) {
                r = import_make_read_only(i->temp_path);
                if (r < 0)
                        return r;
        }

        if (i->force_local)
                (void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);

        r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
        if (r < 0)
                return log_error_errno(r, "Failed to move image into place: %m");

        i->temp_path = mfree(i->temp_path);

        return 0;
}

static int tar_import_fork_tar(TarImport *i) {
        int r;

        assert(i);

        assert(!i->final_path);
        assert(!i->temp_path);
        assert(i->tar_fd < 0);

        i->final_path = strjoin(i->image_root, "/", i->local, NULL);
        if (!i->final_path)
                return log_oom();

        r = tempfn_random(i->final_path, NULL, &i->temp_path);
        if (r < 0)
                return log_oom();

        (void) mkdir_parents_label(i->temp_path, 0700);

        r = btrfs_subvol_make(i->temp_path);
        if (r == -ENOTTY) {
                if (mkdir(i->temp_path, 0755) < 0)
                        return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
        } else if (r < 0)
                return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path);
        else
                (void) import_assign_pool_quota_and_warn(i->temp_path);

        i->tar_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
        if (i->tar_fd < 0)
                return i->tar_fd;

        return 0;
}

static int tar_import_write(const void *p, size_t sz, void *userdata) {
        TarImport *i = userdata;
        int r;

        if (i->grow_machine_directory && i->written_since_last_grow >= GROW_INTERVAL_BYTES) {
                i->written_since_last_grow = 0;
                grow_machine_directory();
        }

        r = loop_write(i->tar_fd, p, sz, false);
        if (r < 0)
                return r;

        i->written_uncompressed += sz;
        i->written_since_last_grow += sz;

        return 0;
}

static int tar_import_process(TarImport *i) {
        ssize_t l;
        int r;

        assert(i);
        assert(i->buffer_size < sizeof(i->buffer));

        l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size);
        if (l < 0) {
                if (errno == EAGAIN)
                        return 0;

                r = log_error_errno(errno, "Failed to read input file: %m");
                goto finish;
        }
        if (l == 0) {
                if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) {
                        log_error("Premature end of file: %m");
                        r = -EIO;
                        goto finish;
                }

                r = tar_import_finish(i);
                goto finish;
        }

        i->buffer_size += l;

        if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) {
                r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size);
                if (r < 0) {
                        log_error("Failed to detect file compression: %m");
                        goto finish;
                }
                if (r == 0) /* Need more data */
                        return 0;

                r = tar_import_fork_tar(i);
                if (r < 0)
                        goto finish;
        }

        r = import_uncompress(&i->compress, i->buffer, i->buffer_size, tar_import_write, i);
        if (r < 0) {
                log_error_errno(r, "Failed to decode and write: %m");
                goto finish;
        }

        i->written_compressed += i->buffer_size;
        i->buffer_size = 0;

        tar_import_report_progress(i);

        return 0;

finish:
        if (i->on_finished)
                i->on_finished(i, r, i->userdata);
        else
                sd_event_exit(i->event, r);

        return 0;
}

static int tar_import_on_input(sd_event_source *s, int fd, uint32_t revents, void *userdata) {
        TarImport *i = userdata;

        return tar_import_process(i);
}

static int tar_import_on_defer(sd_event_source *s, void *userdata) {
        TarImport *i = userdata;

        return tar_import_process(i);
}

int tar_import_start(TarImport *i, int fd, const char *local, bool force_local, bool read_only) {
        int r;

        assert(i);
        assert(fd >= 0);
        assert(local);

        if (!machine_name_is_valid(local))
                return -EINVAL;

        if (i->input_fd >= 0)
                return -EBUSY;

        r = fd_nonblock(fd, true);
        if (r < 0)
                return r;

        r = free_and_strdup(&i->local, local);
        if (r < 0)
                return r;
        i->force_local = force_local;
        i->read_only = read_only;

        if (fstat(fd, &i->st) < 0)
                return -errno;

        r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i);
        if (r == -EPERM) {
                /* This fd does not support epoll, for example because it is a regular file. Busy read in that case */
                r = sd_event_add_defer(i->event, &i->input_event_source, tar_import_on_defer, i);
                if (r < 0)
                        return r;

                r = sd_event_source_set_enabled(i->input_event_source, SD_EVENT_ON);
        }
        if (r < 0)
                return r;

        i->input_fd = fd;
        return r;
}