#include <curl/curl.h>

#include <sys/prctl.h>

#include "set.h"

#include "json.h"

#include "strv.h"

#include "btrfs-util.h"

#include "utf8.h"

#include "mkdir.h"

#include "curl-util.h"

#include "aufs-util.h"

#include "import-util.h"

#include "import-job.h"

#include "import-dkr.h"

struct DkrImport {

sd_event *event;

CurlGlue *glue;

char *index_url;

char *image_root;

ImportJob *images_job;

ImportJob *tags_job;

ImportJob *ancestry_job;

ImportJob *json_job;

ImportJob *layer_job;

char *name;

char *tag;

char *id;

char *response_token;

char **response_registries;

char **ancestry;

unsigned current_ancestry;

DkrImportFinished on_finished;

void *userdata;

char *local;

bool force_local;

char *temp_path;

char *final_path;

pid_t tar_pid;

};

#define PROTOCOL_PREFIX "https://"

#define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"

#define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"

#define LAYERS_MAX 2048

static void dkr_import_job_on_finished(ImportJob *j);

DkrImport* dkr_import_unref(DkrImport *i) {

if (!i)

return NULL;

if (i->tar_pid > 1) {

(void) kill_and_sigcont(i->tar_pid, SIGKILL);

(void) wait_for_terminate(i->tar_pid, NULL);

}

import_job_unref(i->images_job);

import_job_unref(i->tags_job);

import_job_unref(i->ancestry_job);

import_job_unref(i->json_job);

import_job_unref(i->layer_job);

curl_glue_unref(i->glue);

sd_event_unref(i->event);

if (i->temp_path) {

(void) btrfs_subvol_remove(i->temp_path);

(void) rm_rf_dangerous(i->temp_path, false, true, false);

free(i->temp_path);

}

free(i->name);

free(i->tag);

free(i->id);

free(i->response_token);

free(i->response_registries);

strv_free(i->ancestry);

free(i->final_path);

free(i->index_url);

free(i->image_root);

free(i->local);

free(i);

return NULL;

}

int dkr_import_new(

DkrImport **ret,

sd_event *event,

const char *index_url,

const char *image_root,

DkrImportFinished on_finished,

void *userdata) {

_cleanup_(dkr_import_unrefp) DkrImport *i = NULL;

char *e;

int r;

assert(ret);

assert(index_url);

if (!http_url_is_valid(index_url))

return -EINVAL;

i = new0(DkrImport, 1);

if (!i)

return -ENOMEM;

i->on_finished = on_finished;

i->userdata = userdata;

i->image_root = strdup(image_root ?: "/var/lib/machines");

if (!i->image_root)

return -ENOMEM;

i->index_url = strdup(index_url);

if (!i->index_url)

return -ENOMEM;

e = endswith(i->index_url, "/");

if (e)

*e = 0;

if (event)

i->event = sd_event_ref(event);

else {

r = sd_event_default(&i->event);

if (r < 0)

return r;

}

r = curl_glue_new(&i->glue, i->event);

if (r < 0)

return r;

i->glue->on_finished = import_job_curl_on_finished;

i->glue->userdata = i;

*ret = i;

i = NULL;

return 0;

}

static int parse_id(const void *payload, size_t size, char **ret) {

_cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;

union json_value v = {};

void *json_state = NULL;

const char *p;

int t;

assert(payload);

assert(ret);

if (size <= 0)

return -EBADMSG;

if (memchr(payload, 0, size))

return -EBADMSG;

buf = strndup(payload, size);

if (!buf)

return -ENOMEM;

p = buf;

t = json_tokenize(&p, &id, &v, &json_state, NULL);

if (t < 0)

return t;

if (t != JSON_STRING)

return -EBADMSG;

t = json_tokenize(&p, &other, &v, &json_state, NULL);

if (t < 0)

return t;

if (t != JSON_END)

return -EBADMSG;

if (!dkr_id_is_valid(id))

return -EBADMSG;

*ret = id;

id = NULL;

return 0;

}

static int parse_ancestry(const void *payload, size_t size, char ***ret) {

_cleanup_free_ char *buf = NULL;

void *json_state = NULL;

const char *p;

enum {

STATE_BEGIN,

STATE_ITEM,

STATE_COMMA,

STATE_END,

} state = STATE_BEGIN;

_cleanup_strv_free_ char **l = NULL;

size_t n = 0, allocated = 0;

if (size <= 0)

return -EBADMSG;

if (memchr(payload, 0, size))

return -EBADMSG;

buf = strndup(payload, size);

if (!buf)

return -ENOMEM;

p = buf;

for (;;) {

_cleanup_free_ char *str;

union json_value v = {};

int t;

t = json_tokenize(&p, &str, &v, &json_state, NULL);

if (t < 0)

return t;

switch (state) {

case STATE_BEGIN:

if (t == JSON_ARRAY_OPEN)

state = STATE_ITEM;

else

return -EBADMSG;

break;

case STATE_ITEM:

if (t == JSON_STRING) {

if (!dkr_id_is_valid(str))

return -EBADMSG;

if (n+1 > LAYERS_MAX)

return -EFBIG;

if (!GREEDY_REALLOC(l, allocated, n + 2))

return -ENOMEM;

l[n++] = str;

str = NULL;

l[n] = NULL;

state = STATE_COMMA;

} else if (t == JSON_ARRAY_CLOSE)

state = STATE_END;

else

return -EBADMSG;

break;

case STATE_COMMA:

if (t == JSON_COMMA)

state = STATE_ITEM;

else if (t == JSON_ARRAY_CLOSE)

state = STATE_END;

else

return -EBADMSG;

break;

case STATE_END:

if (t == JSON_END) {

if (strv_isempty(l))

return -EBADMSG;

if (!strv_is_uniq(l))

return -EBADMSG;

l = strv_reverse(l);

*ret = l;

l = NULL;

return 0;

} else

return -EBADMSG;

}

}

}

static const char *dkr_import_current_layer(DkrImport *i) {

assert(i);

if (strv_isempty(i->ancestry))

return NULL;

return i->ancestry[i->current_ancestry];

}

static const char *dkr_import_current_base_layer(DkrImport *i) {

assert(i);

if (strv_isempty(i->ancestry))

return NULL;

if (i->current_ancestry <= 0)

return NULL;

return i->ancestry[i->current_ancestry-1];

}

static int dkr_import_add_token(DkrImport *i, ImportJob *j) {

const char *t;

assert(i);

assert(j);

if (i->response_token)

t = strappenda("Authorization: Token ", i->response_token);

else

t = HEADER_TOKEN " true";

j->request_header = curl_slist_new("Accept: application/json", t, NULL);

if (!j->request_header)

return -ENOMEM;

return 0;

}

static bool dkr_import_is_done(DkrImport *i) {

assert(i);

assert(i->images_job);

if (i->images_job->state != IMPORT_JOB_DONE)

return false;

if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)

return false;

if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)

return false;

if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)

return false;

if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)

return false;

if (dkr_import_current_layer(i))

return false;

return true;

}

static int dkr_import_make_local_copy(DkrImport *i) {

int r;

assert(i);

if (!i->local)

return 0;

if (!i->final_path) {

i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);

if (!i->final_path)

return log_oom();

}

r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);

if (r < 0)

return r;

return 0;

}

static int dkr_import_job_on_open_disk(ImportJob *j) {

_cleanup_close_pair_ int pipefd[2] = { -1, -1 };

const char *base;

DkrImport *i;

int r;

assert(j);

assert(j->userdata);

i = j->userdata;

assert(i->layer_job == j);

assert(i->final_path);

assert(!i->temp_path);

assert(i->tar_pid <= 0);

r = tempfn_random(i->final_path, &i->temp_path);

if (r < 0)

return log_oom();

mkdir_parents_label(i->temp_path, 0700);

base = dkr_import_current_base_layer(i);

if (base) {

const char *base_path;

base_path = strappenda(i->image_root, "/.dkr-", base);

r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);

} else

r = btrfs_subvol_make(i->temp_path);

if (r < 0)

return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);

if (pipe2(pipefd, O_CLOEXEC) < 0)

return log_error_errno(errno, "Failed to create pipe for tar: %m");

i->tar_pid = fork();

if (i->tar_pid < 0)

return log_error_errno(errno, "Failed to fork off tar: %m");

if (i->tar_pid == 0) {

int null_fd;

/* Child */

reset_all_signal_handlers();

reset_signal_mask();

assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);

pipefd[1] = safe_close(pipefd[1]);

if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {

log_error_errno(errno, "Failed to dup2() fd: %m");

_exit(EXIT_FAILURE);

}

if (pipefd[0] != STDIN_FILENO)

safe_close(pipefd[0]);

null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);

if (null_fd < 0) {

log_error_errno(errno, "Failed to open /dev/null: %m");

_exit(EXIT_FAILURE);

}

if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {

log_error_errno(errno, "Failed to dup2() fd: %m");

_exit(EXIT_FAILURE);

}

if (null_fd != STDOUT_FILENO)

safe_close(null_fd);

execlp("tar", "tar", "--numeric-owner", "-C", i->temp_path, "-px", NULL);

log_error_errno(errno, "Failed to execute tar: %m");

_exit(EXIT_FAILURE);

}

pipefd[0] = safe_close(pipefd[0]);

j->disk_fd = pipefd[1];

pipefd[1] = -1;

return 0;

}

static int dkr_import_pull_layer(DkrImport *i) {

_cleanup_free_ char *path = NULL;

const char *url, *layer = NULL;

int r;

assert(i);

assert(!i->layer_job);

assert(!i->temp_path);

assert(!i->final_path);

for (;;) {

layer = dkr_import_current_layer(i);

if (!layer)

return 0; /* no more layers */

path = strjoin(i->image_root, "/.dkr-", layer, NULL);

if (!path)

return log_oom();

if (laccess(path, F_OK) < 0) {

if (errno == ENOENT)

break;

return log_error_errno(errno, "Failed to check for container: %m");

}

log_info("Layer %s already exists, skipping.", layer);

i->current_ancestry++;

free(path);

path = NULL;

}

log_info("Pulling layer %s...", layer);

i->final_path = path;

path = NULL;

url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");

r = import_job_new(&i->layer_job, url, i->glue, i);

if (r < 0)

return log_error_errno(r, "Failed to allocate layer job: %m");

r = dkr_import_add_token(i, i->layer_job);

if (r < 0)

return log_oom();

i->layer_job->on_finished = dkr_import_job_on_finished;

i->layer_job->on_open_disk = dkr_import_job_on_open_disk;

r = import_job_begin(i->layer_job);

if (r < 0)

return log_error_errno(r, "Failed to start layer job: %m");

return 0;

}

static void dkr_import_job_on_finished(ImportJob *j) {

DkrImport *i;

int r;

assert(j);

assert(j->userdata);

i = j->userdata;

if (j->error != 0) {

if (j == i->images_job)

log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");

else if (j == i->tags_job)

log_error_errno(j->error, "Failed to retrieve tags list.");

else if (j == i->ancestry_job)

log_error_errno(j->error, "Failed to retrieve ancestry list.");

else if (j == i->json_job)

log_error_errno(j->error, "Failed to retrieve json data.");

else

log_error_errno(j->error, "Failed to retrieve layer data.");

r = j->error;

goto finish;

}

if (i->images_job == j) {

const char *url;

assert(!i->tags_job);

assert(!i->ancestry_job);

assert(!i->json_job);

assert(!i->layer_job);

if (strv_isempty(i->response_registries)) {

r = -EBADMSG;

log_error("Didn't get registry information.");

goto finish;

}

log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);

url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);

r = import_job_new(&i->tags_job, url, i->glue, i);

if (r < 0) {

log_error_errno(r, "Failed to allocate tags job: %m");

goto finish;

}

r = dkr_import_add_token(i, i->tags_job);

if (r < 0) {

log_oom();

goto finish;

}

i->tags_job->on_finished = dkr_import_job_on_finished;

r = import_job_begin(i->tags_job);

if (r < 0) {

log_error_errno(r, "Failed to start tags job: %m");

goto finish;

}

} else if (i->tags_job == j) {

const char *url;

char *id = NULL;

assert(!i->ancestry_job);

assert(!i->json_job);

assert(!i->layer_job);

r = parse_id(j->payload, j->payload_size, &id);

if (r < 0) {

log_error_errno(r, "Failed to parse JSON id.");

goto finish;

}

free(i->id);

i->id = id;

log_info("Tag lookup succeeded, resolved to layer %s.", i->id);

url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");

r = import_job_new(&i->ancestry_job, url, i->glue, i);

if (r < 0) {

log_error_errno(r, "Failed to allocate ancestry job: %m");

goto finish;

}

r = dkr_import_add_token(i, i->ancestry_job);

if (r < 0) {

log_oom();

goto finish;

}

i->ancestry_job->on_finished = dkr_import_job_on_finished;

url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");

r = import_job_new(&i->json_job, url, i->glue, i);

if (r < 0) {

log_error_errno(r, "Failed to allocate json job: %m");

goto finish;

}

r = dkr_import_add_token(i, i->json_job);

if (r < 0) {

log_oom();

goto finish;

}

i->json_job->on_finished = dkr_import_job_on_finished;

r = import_job_begin(i->ancestry_job);

if (r < 0) {

log_error_errno(r, "Failed to start ancestry job: %m");

goto finish;

}

r = import_job_begin(i->json_job);

if (r < 0) {

log_error_errno(r, "Failed to start json job: %m");

goto finish;

}

} else if (i->ancestry_job == j) {

char **ancestry = NULL, **k;

unsigned n;

assert(!i->layer_job);

r = parse_ancestry(j->payload, j->payload_size, &ancestry);

if (r < 0) {

log_error_errno(r, "Failed to parse JSON id.");

goto finish;

}

n = strv_length(ancestry);

if (n <= 0 || !streq(ancestry[n-1], i->id)) {

log_error("Ancestry doesn't end in main layer.");

strv_free(ancestry);

r = -EBADMSG;

goto finish;

}

log_info("Ancestor lookup succeeded, requires layers:\n");

STRV_FOREACH(k, ancestry)

log_info("\t%s", *k);

strv_free(i->ancestry);

i->ancestry = ancestry;

i->current_ancestry = 0;

r = dkr_import_pull_layer(i);

if (r < 0)

goto finish;

} else if (i->layer_job == j) {

assert(i->temp_path);

assert(i->final_path);

j->disk_fd = safe_close(j->disk_fd);

if (i->tar_pid > 0) {

r = wait_for_terminate_and_warn("tar", i->tar_pid, true);

i->tar_pid = 0;

if (r < 0)

goto finish;

}

r = aufs_resolve(i->temp_path);

if (r < 0) {

log_error_errno(r, "Failed to resolve aufs whiteouts: %m");

goto finish;

}

r = btrfs_subvol_set_read_only(i->temp_path, true);

if (r < 0) {

log_error_errno(r, "Failed to mark snapshort read-only: %m");

goto finish;

}

if (rename(i->temp_path, i->final_path) < 0) {

log_error_errno(errno, "Failed to rename snaphsot: %m");

goto finish;

}

log_info("Completed writing to layer %s.", i->final_path);

i->layer_job = import_job_unref(i->layer_job);

free(i->temp_path);

i->temp_path = NULL;

free(i->final_path);

i->final_path = NULL;

i->current_ancestry ++;

r = dkr_import_pull_layer(i);

if (r < 0)

goto finish;

} else if (i->json_job != j)

assert_not_reached("Got finished event for unknown curl object");

if (!dkr_import_is_done(i))

return;

r = dkr_import_make_local_copy(i);

if (r < 0)

goto finish;

r = 0;

finish:

if (i->on_finished)

i->on_finished(i, r, i->userdata);

else

sd_event_exit(i->event, r);

}

static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz) {

_cleanup_free_ char *registry = NULL;

char *token;

DkrImport *i;

int r;

assert(j);

assert(j->userdata);

i = j->userdata;

r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);

if (r < 0)

return log_oom();

if (r > 0) {

free(i->response_token);

i->response_token = token;

return 0;

}

r = curl_header_strdup(header, sz, HEADER_REGISTRY, &registry);

if (r < 0)

return log_oom();

if (r > 0) {

char **l, **k;

l = strv_split(registry, ",");

if (!l)

return log_oom();

STRV_FOREACH(k, l) {

if (!hostname_is_valid(*k)) {

log_error("Registry hostname is not valid.");

strv_free(l);

return -EBADMSG;

}

}

strv_free(i->response_registries);

i->response_registries = l;

}

return 0;

}

int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {

const char *url;

int r;

assert(i);

if (!dkr_name_is_valid(name))

return -EINVAL;

if (tag && !dkr_tag_is_valid(tag))

return -EINVAL;

if (local && !machine_name_is_valid(local))

return -EINVAL;

if (i->images_job)

return -EBUSY;

if (!tag)

tag = "latest";

r = free_and_strdup(&i->local, local);

if (r < 0)

return r;

i->force_local = force_local;

r = free_and_strdup(&i->name, name);

if (r < 0)

return r;

r = free_and_strdup(&i->tag, tag);

if (r < 0)

return r;

url = strappenda(i->index_url, "/v1/repositories/", name, "/images");

r = import_job_new(&i->images_job, url, i->glue, i);

if (r < 0)

return r;

r = dkr_import_add_token(i, i->images_job);

if (r < 0)

return r;

i->images_job->on_finished = dkr_import_job_on_finished;

i->images_job->on_header = dkr_import_job_on_header;

return import_job_begin(i->images_job);

}

bool dkr_name_is_valid(const char *name) {

const char *slash, *p;

if (isempty(name))

return false;

slash = strchr(name, '/');

if (!slash)

return false;

if (!filename_is_valid(slash + 1))

return false;

p = strndupa(name, slash - name);

if (!filename_is_valid(p))

return false;

return true;

}

bool dkr_id_is_valid(const char *id) {

if (!filename_is_valid(id))

return false;

if (!in_charset(id, "0123456789abcdef"))

return false;

return true;

}