diff options
Diffstat (limited to 'src/import')
35 files changed, 8552 insertions, 0 deletions
diff --git a/src/import/curl-util.c b/src/import/curl-util.c new file mode 100644 index 0000000..7db03b2 --- /dev/null +++ b/src/import/curl-util.c @@ -0,0 +1,416 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include "alloc-util.h" +#include "build.h" +#include "curl-util.h" +#include "fd-util.h" +#include "locale-util.h" +#include "string-util.h" + +static void curl_glue_check_finished(CurlGlue *g) { + CURLMsg *msg; + int k = 0; + + assert(g); + + msg = curl_multi_info_read(g->curl, &k); + if (!msg) + return; + + if (msg->msg != CURLMSG_DONE) + return; + + if (g->on_finished) + g->on_finished(g, msg->easy_handle, msg->data.result); +} + +static int curl_glue_on_io(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + CurlGlue *g = userdata; + int action, k = 0, translated_fd; + + assert(s); + assert(g); + + translated_fd = PTR_TO_FD(hashmap_get(g->translate_fds, FD_TO_PTR(fd))); + + if (FLAGS_SET(revents, EPOLLIN | EPOLLOUT)) + action = CURL_POLL_INOUT; + else if (revents & EPOLLIN) + action = CURL_POLL_IN; + else if (revents & EPOLLOUT) + action = CURL_POLL_OUT; + else + action = 0; + + if (curl_multi_socket_action(g->curl, translated_fd, action, &k) != CURLM_OK) + return log_debug_errno(SYNTHETIC_ERRNO(EINVAL), + "Failed to propagate IO event."); + + curl_glue_check_finished(g); + return 0; +} + +static int curl_glue_socket_callback(CURLM *curl, curl_socket_t s, int action, void *userdata, void *socketp) { + sd_event_source *io; + CurlGlue *g = userdata; + uint32_t events = 0; + int r; + + assert(curl); + assert(g); + + io = hashmap_get(g->ios, FD_TO_PTR(s)); + + if (action == CURL_POLL_REMOVE) { + if (io) { + int fd; + + fd = sd_event_source_get_io_fd(io); + assert(fd >= 0); + + sd_event_source_set_enabled(io, SD_EVENT_OFF); + sd_event_source_unref(io); + + hashmap_remove(g->ios, FD_TO_PTR(s)); + hashmap_remove(g->translate_fds, FD_TO_PTR(fd)); + + safe_close(fd); + } + + return 0; + } + + r = hashmap_ensure_allocated(&g->ios, &trivial_hash_ops); + if (r < 0) { + log_oom(); + return -1; + } + + r = hashmap_ensure_allocated(&g->translate_fds, &trivial_hash_ops); + if (r < 0) { + log_oom(); + return -1; + } + + if (action == CURL_POLL_IN) + events = EPOLLIN; + else if (action == CURL_POLL_OUT) + events = EPOLLOUT; + else if (action == CURL_POLL_INOUT) + events = EPOLLIN|EPOLLOUT; + + if (io) { + if (sd_event_source_set_io_events(io, events) < 0) + return -1; + + if (sd_event_source_set_enabled(io, SD_EVENT_ON) < 0) + return -1; + } else { + _cleanup_close_ int fd = -1; + + /* When curl needs to remove an fd from us it closes + * the fd first, and only then calls into us. This is + * nasty, since we cannot pass the fd on to epoll() + * anymore. Hence, duplicate the fds here, and keep a + * copy for epoll which we control after use. */ + + fd = fcntl(s, F_DUPFD_CLOEXEC, 3); + if (fd < 0) + return -1; + + if (sd_event_add_io(g->event, &io, fd, events, curl_glue_on_io, g) < 0) + return -1; + + (void) sd_event_source_set_description(io, "curl-io"); + + r = hashmap_put(g->ios, FD_TO_PTR(s), io); + if (r < 0) { + log_oom(); + sd_event_source_unref(io); + return -1; + } + + r = hashmap_put(g->translate_fds, FD_TO_PTR(fd), FD_TO_PTR(s)); + if (r < 0) { + log_oom(); + hashmap_remove(g->ios, FD_TO_PTR(s)); + sd_event_source_unref(io); + return -1; + } + + fd = -1; + } + + return 0; +} + +static int curl_glue_on_timer(sd_event_source *s, uint64_t usec, void *userdata) { + CurlGlue *g = userdata; + int k = 0; + + assert(s); + assert(g); + + if (curl_multi_socket_action(g->curl, CURL_SOCKET_TIMEOUT, 0, &k) != CURLM_OK) + return log_debug_errno(SYNTHETIC_ERRNO(EINVAL), + "Failed to propagate timeout."); + + curl_glue_check_finished(g); + return 0; +} + +static int curl_glue_timer_callback(CURLM *curl, long timeout_ms, void *userdata) { + CurlGlue *g = userdata; + usec_t usec; + + assert(curl); + assert(g); + + if (timeout_ms < 0) { + if (g->timer) { + if (sd_event_source_set_enabled(g->timer, SD_EVENT_OFF) < 0) + return -1; + } + + return 0; + } + + usec = now(clock_boottime_or_monotonic()) + (usec_t) timeout_ms * USEC_PER_MSEC + USEC_PER_MSEC - 1; + + if (g->timer) { + if (sd_event_source_set_time(g->timer, usec) < 0) + return -1; + + if (sd_event_source_set_enabled(g->timer, SD_EVENT_ONESHOT) < 0) + return -1; + } else { + if (sd_event_add_time(g->event, &g->timer, clock_boottime_or_monotonic(), usec, 0, curl_glue_on_timer, g) < 0) + return -1; + + (void) sd_event_source_set_description(g->timer, "curl-timer"); + } + + return 0; +} + +CurlGlue *curl_glue_unref(CurlGlue *g) { + sd_event_source *io; + + if (!g) + return NULL; + + if (g->curl) + curl_multi_cleanup(g->curl); + + while ((io = hashmap_steal_first(g->ios))) { + int fd; + + fd = sd_event_source_get_io_fd(io); + assert(fd >= 0); + + hashmap_remove(g->translate_fds, FD_TO_PTR(fd)); + + safe_close(fd); + sd_event_source_unref(io); + } + + hashmap_free(g->ios); + + sd_event_source_unref(g->timer); + sd_event_unref(g->event); + return mfree(g); +} + +int curl_glue_new(CurlGlue **glue, sd_event *event) { + _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL; + _cleanup_(curl_multi_cleanupp) CURL *c = NULL; + _cleanup_(sd_event_unrefp) sd_event *e = NULL; + int r; + + if (event) + e = sd_event_ref(event); + else { + r = sd_event_default(&e); + if (r < 0) + return r; + } + + c = curl_multi_init(); + if (!c) + return -ENOMEM; + + g = new(CurlGlue, 1); + if (!g) + return -ENOMEM; + + *g = (CurlGlue) { + .event = TAKE_PTR(e), + .curl = TAKE_PTR(c), + }; + + if (curl_multi_setopt(g->curl, CURLMOPT_SOCKETDATA, g) != CURLM_OK) + return -EINVAL; + + if (curl_multi_setopt(g->curl, CURLMOPT_SOCKETFUNCTION, curl_glue_socket_callback) != CURLM_OK) + return -EINVAL; + + if (curl_multi_setopt(g->curl, CURLMOPT_TIMERDATA, g) != CURLM_OK) + return -EINVAL; + + if (curl_multi_setopt(g->curl, CURLMOPT_TIMERFUNCTION, curl_glue_timer_callback) != CURLM_OK) + return -EINVAL; + + *glue = TAKE_PTR(g); + + return 0; +} + +int curl_glue_make(CURL **ret, const char *url, void *userdata) { + _cleanup_(curl_easy_cleanupp) CURL *c = NULL; + const char *useragent; + + assert(ret); + assert(url); + + c = curl_easy_init(); + if (!c) + return -ENOMEM; + + /* curl_easy_setopt(c, CURLOPT_VERBOSE, 1L); */ + + if (curl_easy_setopt(c, CURLOPT_URL, url) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(c, CURLOPT_PRIVATE, userdata) != CURLE_OK) + return -EIO; + + useragent = strjoina(program_invocation_short_name, "/" GIT_VERSION); + if (curl_easy_setopt(c, CURLOPT_USERAGENT, useragent) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) + return -EIO; + + *ret = TAKE_PTR(c); + return 0; +} + +int curl_glue_add(CurlGlue *g, CURL *c) { + assert(g); + assert(c); + + if (curl_multi_add_handle(g->curl, c) != CURLM_OK) + return -EIO; + + return 0; +} + +void curl_glue_remove_and_free(CurlGlue *g, CURL *c) { + assert(g); + + if (!c) + return; + + if (g->curl) + curl_multi_remove_handle(g->curl, c); + + curl_easy_cleanup(c); +} + +struct curl_slist *curl_slist_new(const char *first, ...) { + struct curl_slist *l; + va_list ap; + + if (!first) + return NULL; + + l = curl_slist_append(NULL, first); + if (!l) + return NULL; + + va_start(ap, first); + + for (;;) { + struct curl_slist *n; + const char *i; + + i = va_arg(ap, const char*); + if (!i) + break; + + n = curl_slist_append(l, i); + if (!n) { + va_end(ap); + curl_slist_free_all(l); + return NULL; + } + + l = n; + } + + va_end(ap); + return l; +} + +int curl_header_strdup(const void *contents, size_t sz, const char *field, char **value) { + const char *p; + char *s; + + p = memory_startswith_no_case(contents, sz, field); + if (!p) + return 0; + + sz -= p - (const char*) contents; + + if (memchr(p, 0, sz)) + return 0; + + /* Skip over preceeding whitespace */ + while (sz > 0 && strchr(WHITESPACE, p[0])) { + p++; + sz--; + } + + /* Truncate trailing whitespace */ + while (sz > 0 && strchr(WHITESPACE, p[sz-1])) + sz--; + + s = strndup(p, sz); + if (!s) + return -ENOMEM; + + *value = s; + return 1; +} + +int curl_parse_http_time(const char *t, usec_t *ret) { + _cleanup_(freelocalep) locale_t loc = (locale_t) 0; + const char *e; + struct tm tm; + time_t v; + + assert(t); + assert(ret); + + loc = newlocale(LC_TIME_MASK, "C", (locale_t) 0); + if (loc == (locale_t) 0) + return -errno; + + /* RFC822 */ + e = strptime_l(t, "%a, %d %b %Y %H:%M:%S %Z", &tm, loc); + if (!e || *e != 0) + /* RFC 850 */ + e = strptime_l(t, "%A, %d-%b-%y %H:%M:%S %Z", &tm, loc); + if (!e || *e != 0) + /* ANSI C */ + e = strptime_l(t, "%a %b %d %H:%M:%S %Y", &tm, loc); + if (!e || *e != 0) + return -EINVAL; + + v = timegm(&tm); + if (v == (time_t) -1) + return -EINVAL; + + *ret = (usec_t) v * USEC_PER_SEC; + return 0; +} diff --git a/src/import/curl-util.h b/src/import/curl-util.h new file mode 100644 index 0000000..a03e844 --- /dev/null +++ b/src/import/curl-util.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include <curl/curl.h> +#include <sys/types.h> + +#include "sd-event.h" + +#include "hashmap.h" + +typedef struct CurlGlue CurlGlue; + +struct CurlGlue { + sd_event *event; + CURLM *curl; + sd_event_source *timer; + Hashmap *ios; + Hashmap *translate_fds; + + void (*on_finished)(CurlGlue *g, CURL *curl, CURLcode code); + void *userdata; +}; + +int curl_glue_new(CurlGlue **glue, sd_event *event); +CurlGlue* curl_glue_unref(CurlGlue *glue); + +DEFINE_TRIVIAL_CLEANUP_FUNC(CurlGlue*, curl_glue_unref); + +int curl_glue_make(CURL **ret, const char *url, void *userdata); +int curl_glue_add(CurlGlue *g, CURL *c); +void curl_glue_remove_and_free(CurlGlue *g, CURL *c); + +struct curl_slist *curl_slist_new(const char *first, ...) _sentinel_; +int curl_header_strdup(const void *contents, size_t sz, const char *field, char **value); +int curl_parse_http_time(const char *t, usec_t *ret); + +DEFINE_TRIVIAL_CLEANUP_FUNC(CURL*, curl_easy_cleanup); +DEFINE_TRIVIAL_CLEANUP_FUNC(CURL*, curl_multi_cleanup); +DEFINE_TRIVIAL_CLEANUP_FUNC(struct curl_slist*, curl_slist_free_all); diff --git a/src/import/export-raw.c b/src/import/export-raw.c new file mode 100644 index 0000000..6a02b47 --- /dev/null +++ b/src/import/export-raw.c @@ -0,0 +1,332 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <sys/sendfile.h> + +/* When we include libgen.h because we need dirname() we immediately + * undefine basename() since libgen.h defines it as a macro to the POSIX + * version which is really broken. We prefer GNU basename(). */ +#include <libgen.h> +#undef basename + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "export-raw.h" +#include "fd-util.h" +#include "fs-util.h" +#include "import-common.h" +#include "missing.h" +#include "ratelimit.h" +#include "stat-util.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +#define COPY_BUFFER_SIZE (16*1024) + +struct RawExport { + sd_event *event; + + RawExportFinished on_finished; + void *userdata; + + char *path; + + int input_fd; + int output_fd; + + ImportCompress compress; + + sd_event_source *output_event_source; + + void *buffer; + size_t buffer_size; + size_t buffer_allocated; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + unsigned last_percent; + RateLimit progress_rate_limit; + + struct stat st; + + bool eof; + bool tried_reflink; + bool tried_sendfile; +}; + +RawExport *raw_export_unref(RawExport *e) { + if (!e) + return NULL; + + sd_event_source_unref(e->output_event_source); + + import_compress_free(&e->compress); + + sd_event_unref(e->event); + + safe_close(e->input_fd); + + free(e->buffer); + free(e->path); + return mfree(e); +} + +int raw_export_new( + RawExport **ret, + sd_event *event, + RawExportFinished on_finished, + void *userdata) { + + _cleanup_(raw_export_unrefp) RawExport *e = NULL; + int r; + + assert(ret); + + e = new(RawExport, 1); + if (!e) + return -ENOMEM; + + *e = (RawExport) { + .output_fd = -1, + .input_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .last_percent = (unsigned) -1, + }; + + RATELIMIT_INIT(e->progress_rate_limit, 100 * USEC_PER_MSEC, 1); + + if (event) + e->event = sd_event_ref(event); + else { + r = sd_event_default(&e->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(e); + + return 0; +} + +static void raw_export_report_progress(RawExport *e) { + unsigned percent; + assert(e); + + if (e->written_uncompressed >= (uint64_t) e->st.st_size) + percent = 100; + else + percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / (uint64_t) e->st.st_size); + + if (percent == e->last_percent) + return; + + if (!ratelimit_below(&e->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Exported %u%%.", percent); + + e->last_percent = percent; +} + +static int raw_export_process(RawExport *e) { + ssize_t l; + int r; + + assert(e); + + if (!e->tried_reflink && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + /* If we shall take an uncompressed snapshot we can + * reflink source to destination directly. Let's see + * if this works. */ + + r = btrfs_reflink(e->input_fd, e->output_fd); + if (r >= 0) { + r = 0; + goto finish; + } + + e->tried_reflink = true; + } + + if (!e->tried_sendfile && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + l = sendfile(e->output_fd, e->input_fd, NULL, COPY_BUFFER_SIZE); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + e->tried_sendfile = true; + } else if (l == 0) { + r = 0; + goto finish; + } else { + e->written_uncompressed += l; + e->written_compressed += l; + + raw_export_report_progress(e); + + return 0; + } + } + + while (e->buffer_size <= 0) { + uint8_t input[COPY_BUFFER_SIZE]; + + if (e->eof) { + r = 0; + goto finish; + } + + l = read(e->input_fd, input, sizeof(input)); + if (l < 0) { + r = log_error_errno(errno, "Failed to read raw file: %m"); + goto finish; + } + + if (l == 0) { + e->eof = true; + r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } else { + e->written_uncompressed += l; + r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } + if (r < 0) { + r = log_error_errno(r, "Failed to encode: %m"); + goto finish; + } + } + + l = write(e->output_fd, e->buffer, e->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to write output file: %m"); + goto finish; + } + + assert((size_t) l <= e->buffer_size); + memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l); + e->buffer_size -= l; + e->written_compressed += l; + + raw_export_report_progress(e); + + return 0; + +finish: + if (r >= 0) { + (void) copy_times(e->input_fd, e->output_fd); + (void) copy_xattr(e->input_fd, e->output_fd); + } + + if (e->on_finished) + e->on_finished(e, r, e->userdata); + else + sd_event_exit(e->event, r); + + return 0; +} + +static int raw_export_on_output(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + RawExport *i = userdata; + + return raw_export_process(i); +} + +static int raw_export_on_defer(sd_event_source *s, void *userdata) { + RawExport *i = userdata; + + return raw_export_process(i); +} + +static int reflink_snapshot(int fd, const char *path) { + int new_fd, r; + + new_fd = open_parent(path, O_TMPFILE|O_CLOEXEC|O_RDWR, 0600); + if (new_fd < 0) { + _cleanup_free_ char *t = NULL; + + r = tempfn_random(path, NULL, &t); + if (r < 0) + return r; + + new_fd = open(t, O_CLOEXEC|O_CREAT|O_NOCTTY|O_RDWR, 0600); + if (new_fd < 0) + return -errno; + + (void) unlink(t); + } + + r = btrfs_reflink(fd, new_fd); + if (r < 0) { + safe_close(new_fd); + return r; + } + + return new_fd; +} + +int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType compress) { + _cleanup_close_ int sfd = -1, tfd = -1; + int r; + + assert(e); + assert(path); + assert(fd >= 0); + assert(compress < _IMPORT_COMPRESS_TYPE_MAX); + assert(compress != IMPORT_COMPRESS_UNKNOWN); + + if (e->output_fd >= 0) + return -EBUSY; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&e->path, path); + if (r < 0) + return r; + + sfd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (sfd < 0) + return -errno; + + if (fstat(sfd, &e->st) < 0) + return -errno; + r = stat_verify_regular(&e->st); + if (r < 0) + return r; + + /* Try to take a reflink snapshot of the file, if we can t make the export atomic */ + tfd = reflink_snapshot(sfd, path); + if (tfd >= 0) + e->input_fd = TAKE_FD(tfd); + else + e->input_fd = TAKE_FD(sfd); + + r = import_compress_init(&e->compress, compress); + if (r < 0) + return r; + + r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, raw_export_on_output, e); + if (r == -EPERM) { + r = sd_event_add_defer(e->event, &e->output_event_source, raw_export_on_defer, e); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + e->output_fd = fd; + return r; +} diff --git a/src/import/export-raw.h b/src/import/export-raw.h new file mode 100644 index 0000000..196b8ef --- /dev/null +++ b/src/import/export-raw.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include "sd-event.h" + +#include "import-compress.h" +#include "macro.h" + +typedef struct RawExport RawExport; + +typedef void (*RawExportFinished)(RawExport *export, int error, void *userdata); + +int raw_export_new(RawExport **export, sd_event *event, RawExportFinished on_finished, void *userdata); +RawExport* raw_export_unref(RawExport *export); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawExport*, raw_export_unref); + +int raw_export_start(RawExport *export, const char *path, int fd, ImportCompressType compress); diff --git a/src/import/export-tar.c b/src/import/export-tar.c new file mode 100644 index 0000000..ed54676 --- /dev/null +++ b/src/import/export-tar.c @@ -0,0 +1,331 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "export-tar.h" +#include "fd-util.h" +#include "import-common.h" +#include "process-util.h" +#include "ratelimit.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +#define COPY_BUFFER_SIZE (16*1024) + +struct TarExport { + sd_event *event; + + TarExportFinished on_finished; + void *userdata; + + char *path; + char *temp_path; + + int output_fd; + int tar_fd; + + ImportCompress compress; + + sd_event_source *output_event_source; + + void *buffer; + size_t buffer_size; + size_t buffer_allocated; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + pid_t tar_pid; + + struct stat st; + uint64_t quota_referenced; + + unsigned last_percent; + RateLimit progress_rate_limit; + + bool eof; + bool tried_splice; +}; + +TarExport *tar_export_unref(TarExport *e) { + if (!e) + return NULL; + + sd_event_source_unref(e->output_event_source); + + if (e->tar_pid > 1) { + (void) kill_and_sigcont(e->tar_pid, SIGKILL); + (void) wait_for_terminate(e->tar_pid, NULL); + } + + if (e->temp_path) { + (void) btrfs_subvol_remove(e->temp_path, BTRFS_REMOVE_QUOTA); + free(e->temp_path); + } + + import_compress_free(&e->compress); + + sd_event_unref(e->event); + + safe_close(e->tar_fd); + + free(e->buffer); + free(e->path); + return mfree(e); +} + +int tar_export_new( + TarExport **ret, + sd_event *event, + TarExportFinished on_finished, + void *userdata) { + + _cleanup_(tar_export_unrefp) TarExport *e = NULL; + int r; + + assert(ret); + + e = new(TarExport, 1); + if (!e) + return -ENOMEM; + + *e = (TarExport) { + .output_fd = -1, + .tar_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .quota_referenced = (uint64_t) -1, + .last_percent = (unsigned) -1, + }; + + RATELIMIT_INIT(e->progress_rate_limit, 100 * USEC_PER_MSEC, 1); + + if (event) + e->event = sd_event_ref(event); + else { + r = sd_event_default(&e->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(e); + + return 0; +} + +static void tar_export_report_progress(TarExport *e) { + unsigned percent; + assert(e); + + /* Do we have any quota info? If not, we don't know anything about the progress */ + if (e->quota_referenced == (uint64_t) -1) + return; + + if (e->written_uncompressed >= e->quota_referenced) + percent = 100; + else + percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / e->quota_referenced); + + if (percent == e->last_percent) + return; + + if (!ratelimit_below(&e->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Exported %u%%.", percent); + + e->last_percent = percent; +} + +static int tar_export_finish(TarExport *e) { + int r; + + assert(e); + assert(e->tar_fd >= 0); + + if (e->tar_pid > 0) { + r = wait_for_terminate_and_check("tar", e->tar_pid, WAIT_LOG); + e->tar_pid = 0; + if (r < 0) + return r; + if (r != EXIT_SUCCESS) + return -EPROTO; + } + + e->tar_fd = safe_close(e->tar_fd); + + return 0; +} + +static int tar_export_process(TarExport *e) { + ssize_t l; + int r; + + assert(e); + + if (!e->tried_splice && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + l = splice(e->tar_fd, NULL, e->output_fd, NULL, COPY_BUFFER_SIZE, 0); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + e->tried_splice = true; + } else if (l == 0) { + r = tar_export_finish(e); + goto finish; + } else { + e->written_uncompressed += l; + e->written_compressed += l; + + tar_export_report_progress(e); + + return 0; + } + } + + while (e->buffer_size <= 0) { + uint8_t input[COPY_BUFFER_SIZE]; + + if (e->eof) { + r = tar_export_finish(e); + goto finish; + } + + l = read(e->tar_fd, input, sizeof(input)); + if (l < 0) { + r = log_error_errno(errno, "Failed to read tar file: %m"); + goto finish; + } + + if (l == 0) { + e->eof = true; + r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } else { + e->written_uncompressed += l; + r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } + if (r < 0) { + r = log_error_errno(r, "Failed to encode: %m"); + goto finish; + } + } + + l = write(e->output_fd, e->buffer, e->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to write output file: %m"); + goto finish; + } + + assert((size_t) l <= e->buffer_size); + memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l); + e->buffer_size -= l; + e->written_compressed += l; + + tar_export_report_progress(e); + + return 0; + +finish: + if (e->on_finished) + e->on_finished(e, r, e->userdata); + else + sd_event_exit(e->event, r); + + return 0; +} + +static int tar_export_on_output(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + TarExport *i = userdata; + + return tar_export_process(i); +} + +static int tar_export_on_defer(sd_event_source *s, void *userdata) { + TarExport *i = userdata; + + return tar_export_process(i); +} + +int tar_export_start(TarExport *e, const char *path, int fd, ImportCompressType compress) { + _cleanup_close_ int sfd = -1; + int r; + + assert(e); + assert(path); + assert(fd >= 0); + assert(compress < _IMPORT_COMPRESS_TYPE_MAX); + assert(compress != IMPORT_COMPRESS_UNKNOWN); + + if (e->output_fd >= 0) + return -EBUSY; + + sfd = open(path, O_DIRECTORY|O_RDONLY|O_NOCTTY|O_CLOEXEC); + if (sfd < 0) + return -errno; + + if (fstat(sfd, &e->st) < 0) + return -errno; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&e->path, path); + if (r < 0) + return r; + + e->quota_referenced = (uint64_t) -1; + + if (e->st.st_ino == 256) { /* might be a btrfs subvolume? */ + BtrfsQuotaInfo q; + + r = btrfs_subvol_get_subtree_quota_fd(sfd, 0, &q); + if (r >= 0) + e->quota_referenced = q.referenced; + + e->temp_path = mfree(e->temp_path); + + r = tempfn_random(path, NULL, &e->temp_path); + if (r < 0) + return r; + + /* Let's try to make a snapshot, if we can, so that the export is atomic */ + r = btrfs_subvol_snapshot_fd(sfd, e->temp_path, BTRFS_SNAPSHOT_READ_ONLY|BTRFS_SNAPSHOT_RECURSIVE); + if (r < 0) { + log_debug_errno(r, "Couldn't create snapshot %s of %s, not exporting atomically: %m", e->temp_path, path); + e->temp_path = mfree(e->temp_path); + } + } + + r = import_compress_init(&e->compress, compress); + if (r < 0) + return r; + + r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, tar_export_on_output, e); + if (r == -EPERM) { + r = sd_event_add_defer(e->event, &e->output_event_source, tar_export_on_defer, e); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + e->tar_fd = import_fork_tar_c(e->temp_path ?: e->path, &e->tar_pid); + if (e->tar_fd < 0) { + e->output_event_source = sd_event_source_unref(e->output_event_source); + return e->tar_fd; + } + + e->output_fd = fd; + return r; +} diff --git a/src/import/export-tar.h b/src/import/export-tar.h new file mode 100644 index 0000000..6abb7d3 --- /dev/null +++ b/src/import/export-tar.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include "sd-event.h" + +#include "import-compress.h" +#include "macro.h" + +typedef struct TarExport TarExport; + +typedef void (*TarExportFinished)(TarExport *export, int error, void *userdata); + +int tar_export_new(TarExport **export, sd_event *event, TarExportFinished on_finished, void *userdata); +TarExport* tar_export_unref(TarExport *export); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarExport*, tar_export_unref); + +int tar_export_start(TarExport *export, const char *path, int fd, ImportCompressType compress); diff --git a/src/import/export.c b/src/import/export.c new file mode 100644 index 0000000..4907106 --- /dev/null +++ b/src/import/export.c @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <getopt.h> + +#include "sd-event.h" +#include "sd-id128.h" + +#include "alloc-util.h" +#include "export-raw.h" +#include "export-tar.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-util.h" +#include "machine-image.h" +#include "main-func.h" +#include "signal-util.h" +#include "string-util.h" +#include "verbs.h" + +static ImportCompressType arg_compress = IMPORT_COMPRESS_UNKNOWN; + +static void determine_compression_from_filename(const char *p) { + + if (arg_compress != IMPORT_COMPRESS_UNKNOWN) + return; + + if (!p) { + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; + return; + } + + if (endswith(p, ".xz")) + arg_compress = IMPORT_COMPRESS_XZ; + else if (endswith(p, ".gz")) + arg_compress = IMPORT_COMPRESS_GZIP; + else if (endswith(p, ".bz2")) + arg_compress = IMPORT_COMPRESS_BZIP2; + else + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; +} + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +static void on_tar_finished(TarExport *export, int error, void *userdata) { + sd_event *event = userdata; + assert(export); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int export_tar(int argc, char *argv[], void *userdata) { + _cleanup_(tar_export_unrefp) TarExport *export = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(image_unrefp) Image *image = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (machine_name_is_valid(argv[1])) { + r = image_find(IMAGE_MACHINE, argv[1], &image); + if (r == -ENOENT) + return log_error_errno(r, "Machine image %s not found.", argv[1]); + if (r < 0) + return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + + local = image->path; + } else + local = argv[1]; + + if (argc >= 3) + path = argv[2]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + determine_compression_from_filename(path); + + if (path) { + open_fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open tar image for export: %m"); + + fd = open_fd; + + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDOUT_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = tar_export_new(&export, event, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate exporter: %m"); + + r = tar_export_start(export, local, fd, arg_compress); + if (r < 0) + return log_error_errno(r, "Failed to export image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawExport *export, int error, void *userdata) { + sd_event *event = userdata; + assert(export); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int export_raw(int argc, char *argv[], void *userdata) { + _cleanup_(raw_export_unrefp) RawExport *export = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(image_unrefp) Image *image = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (machine_name_is_valid(argv[1])) { + r = image_find(IMAGE_MACHINE, argv[1], &image); + if (r == -ENOENT) + return log_error_errno(r, "Machine image %s not found.", argv[1]); + if (r < 0) + return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + + local = image->path; + } else + local = argv[1]; + + if (argc >= 3) + path = argv[2]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + determine_compression_from_filename(path); + + if (path) { + open_fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open raw image for export: %m"); + + fd = open_fd; + + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDOUT_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = raw_export_new(&export, event, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate exporter: %m"); + + r = raw_export_start(export, local, fd, arg_compress); + if (r < 0) + return log_error_errno(r, "Failed to export image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%s [OPTIONS...] {COMMAND} ...\n\n" + "Export container or virtual machine images.\n\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --format=FORMAT Select format\n\n" + "Commands:\n" + " tar NAME [FILE] Export a TAR image\n" + " raw NAME [FILE] Export a RAW image\n", + program_invocation_short_name); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORMAT, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "format", required_argument, NULL, ARG_FORMAT }, + {} + }; + + int c; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORMAT: + if (streq(optarg, "uncompressed")) + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; + else if (streq(optarg, "xz")) + arg_compress = IMPORT_COMPRESS_XZ; + else if (streq(optarg, "gzip")) + arg_compress = IMPORT_COMPRESS_GZIP; + else if (streq(optarg, "bzip2")) + arg_compress = IMPORT_COMPRESS_BZIP2; + else + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Unknown format: %s", optarg); + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached("Unhandled option"); + } + + return 1; +} + +static int export_main(int argc, char *argv[]) { + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, export_tar }, + { "raw", 2, 3, 0, export_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +static int run(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + return r; + + (void) ignore_signals(SIGPIPE, -1); + + return export_main(argc, argv); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/import-common.c b/src/import/import-common.c new file mode 100644 index 0000000..89f0301 --- /dev/null +++ b/src/import/import-common.c @@ -0,0 +1,259 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <sched.h> +#include <sys/prctl.h> +#include <sys/stat.h> +#include <unistd.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "capability-util.h" +#include "dirent-util.h" +#include "fd-util.h" +#include "fileio.h" +#include "fs-util.h" +#include "import-common.h" +#include "os-util.h" +#include "process-util.h" +#include "signal-util.h" +#include "tmpfile-util.h" +#include "util.h" + +int import_make_read_only_fd(int fd) { + int r; + + assert(fd >= 0); + + /* First, let's make this a read-only subvolume if it refers + * to a subvolume */ + r = btrfs_subvol_set_read_only_fd(fd, true); + if (IN_SET(r, -ENOTTY, -ENOTDIR, -EINVAL)) { + struct stat st; + + /* This doesn't refer to a subvolume, or the file + * system isn't even btrfs. In that, case fall back to + * chmod()ing */ + + r = fstat(fd, &st); + if (r < 0) + return log_error_errno(errno, "Failed to stat temporary image: %m"); + + /* Drop "w" flag */ + if (fchmod(fd, st.st_mode & 07555) < 0) + return log_error_errno(errno, "Failed to chmod() final image: %m"); + + return 0; + + } else if (r < 0) + return log_error_errno(r, "Failed to make subvolume read-only: %m"); + + return 0; +} + +int import_make_read_only(const char *path) { + _cleanup_close_ int fd = 1; + + fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + + return import_make_read_only_fd(fd); +} + +int import_fork_tar_x(const char *path, pid_t *ret) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + pid_t pid; + int r; + + assert(path); + assert(ret); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return log_error_errno(errno, "Failed to create pipe for tar: %m"); + + r = safe_fork("(tar)", FORK_RESET_SIGNALS|FORK_DEATHSIG|FORK_LOG, &pid); + if (r < 0) + return r; + if (r == 0) { + uint64_t retain = + (1ULL << CAP_CHOWN) | + (1ULL << CAP_FOWNER) | + (1ULL << CAP_FSETID) | + (1ULL << CAP_MKNOD) | + (1ULL << CAP_SETFCAP) | + (1ULL << CAP_DAC_OVERRIDE); + + /* Child */ + + pipefd[1] = safe_close(pipefd[1]); + + r = rearrange_stdio(pipefd[0], -1, STDERR_FILENO); + if (r < 0) { + log_error_errno(r, "Failed to rearrange stdin/stdout: %m"); + _exit(EXIT_FAILURE); + } + + if (unshare(CLONE_NEWNET) < 0) + log_error_errno(errno, "Failed to lock tar into network namespace, ignoring: %m"); + + r = capability_bounding_set_drop(retain, true); + if (r < 0) + log_error_errno(r, "Failed to drop capabilities, ignoring: %m"); + + execlp("tar", "tar", "--numeric-owner", "-C", path, "-px", "--xattrs", "--xattrs-include=*", NULL); + log_error_errno(errno, "Failed to execute tar: %m"); + _exit(EXIT_FAILURE); + } + + *ret = pid; + + return TAKE_FD(pipefd[1]); +} + +int import_fork_tar_c(const char *path, pid_t *ret) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + pid_t pid; + int r; + + assert(path); + assert(ret); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return log_error_errno(errno, "Failed to create pipe for tar: %m"); + + r = safe_fork("(tar)", FORK_RESET_SIGNALS|FORK_DEATHSIG|FORK_LOG, &pid); + if (r < 0) + return r; + if (r == 0) { + uint64_t retain = (1ULL << CAP_DAC_OVERRIDE); + + /* Child */ + + pipefd[0] = safe_close(pipefd[0]); + + r = rearrange_stdio(-1, pipefd[1], STDERR_FILENO); + if (r < 0) { + log_error_errno(r, "Failed to rearrange stdin/stdout: %m"); + _exit(EXIT_FAILURE); + } + + if (unshare(CLONE_NEWNET) < 0) + log_error_errno(errno, "Failed to lock tar into network namespace, ignoring: %m"); + + r = capability_bounding_set_drop(retain, true); + if (r < 0) + log_error_errno(r, "Failed to drop capabilities, ignoring: %m"); + + execlp("tar", "tar", "-C", path, "-c", "--xattrs", "--xattrs-include=*", ".", NULL); + log_error_errno(errno, "Failed to execute tar: %m"); + _exit(EXIT_FAILURE); + } + + *ret = pid; + + return TAKE_FD(pipefd[0]); +} + +int import_mangle_os_tree(const char *path) { + _cleanup_closedir_ DIR *d = NULL, *cd = NULL; + _cleanup_free_ char *child = NULL, *t = NULL; + const char *joined; + struct dirent *de; + int r; + + assert(path); + + /* Some tarballs contain a single top-level directory that contains the actual OS directory tree. Try to + * recognize this, and move the tree one level up. */ + + r = path_is_os_tree(path); + if (r < 0) + return log_error_errno(r, "Failed to determine whether '%s' is an OS tree: %m", path); + if (r > 0) { + log_debug("Directory tree '%s' is a valid OS tree.", path); + return 0; + } + + log_debug("Directory tree '%s' is not recognizable as OS tree, checking whether to rearrange it.", path); + + d = opendir(path); + if (!d) + return log_error_errno(r, "Failed to open directory '%s': %m", path); + + errno = 0; + de = readdir_no_dot(d); + if (!de) { + if (errno != 0) + return log_error_errno(errno, "Failed to iterate through directory '%s': %m", path); + + log_debug("Directory '%s' is empty, leaving it as it is.", path); + return 0; + } + + child = strdup(de->d_name); + if (!child) + return log_oom(); + + errno = 0; + de = readdir_no_dot(d); + if (de) { + if (errno != 0) + return log_error_errno(errno, "Failed to iterate through directory '%s': %m", path); + + log_debug("Directory '%s' does not look like a directory tree, and has multiple children, leaving as it is.", path); + return 0; + } + + joined = strjoina(path, "/", child); + r = path_is_os_tree(joined); + if (r == -ENOTDIR) { + log_debug("Directory '%s' does not look like a directory tree, and contains a single regular file only, leaving as it is.", path); + return 0; + } + if (r < 0) + return log_error_errno(r, "Failed to determine whether '%s' is an OS tree: %m", joined); + if (r == 0) { + log_debug("Neither '%s' nor '%s' is a valid OS tree, leaving them as they are.", path, joined); + return 0; + } + + /* Nice, we have checked now: + * + * 1. The top-level directory does not qualify as OS tree + * 1. The top-level directory only contains one item + * 2. That item is a directory + * 3. And that directory qualifies as OS tree + * + * Let's now rearrange things, moving everything in the inner directory one level up */ + + cd = xopendirat(dirfd(d), child, O_NOFOLLOW); + if (!cd) + return log_error_errno(errno, "Can't open directory '%s': %m", joined); + + log_info("Rearranging '%s', moving OS tree one directory up.", joined); + + /* Let's rename the child to an unguessable name so that we can be sure all files contained in it can be + * safely moved up and won't collide with the name. */ + r = tempfn_random(child, NULL, &t); + if (r < 0) + return log_oom(); + r = rename_noreplace(dirfd(d), child, dirfd(d), t); + if (r < 0) + return log_error_errno(r, "Unable to rename '%s' to '%s/%s': %m", joined, path, t); + + FOREACH_DIRENT_ALL(de, cd, return log_error_errno(errno, "Failed to iterate through directory '%s': %m", joined)) { + if (dot_or_dot_dot(de->d_name)) + continue; + + r = rename_noreplace(dirfd(cd), de->d_name, dirfd(d), de->d_name); + if (r < 0) + return log_error_errno(r, "Unable to move '%s/%s/%s' to '%s/%s': %m", path, t, de->d_name, path, de->d_name); + } + + if (unlinkat(dirfd(d), t, AT_REMOVEDIR) < 0) + return log_error_errno(errno, "Failed to remove temporary directory '%s/%s': %m", path, t); + + log_info("Successfully rearranged OS tree."); + + return 0; +} diff --git a/src/import/import-common.h b/src/import/import-common.h new file mode 100644 index 0000000..94d224f --- /dev/null +++ b/src/import/import-common.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include <sys/types.h> + +int import_make_read_only_fd(int fd); +int import_make_read_only(const char *path); + +int import_fork_tar_c(const char *path, pid_t *ret); +int import_fork_tar_x(const char *path, pid_t *ret); + +int import_mangle_os_tree(const char *path); diff --git a/src/import/import-compress.c b/src/import/import-compress.c new file mode 100644 index 0000000..3fbd067 --- /dev/null +++ b/src/import/import-compress.c @@ -0,0 +1,466 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include "import-compress.h" +#include "string-table.h" +#include "util.h" + +void import_compress_free(ImportCompress *c) { + assert(c); + + if (c->type == IMPORT_COMPRESS_XZ) + lzma_end(&c->xz); + else if (c->type == IMPORT_COMPRESS_GZIP) { + if (c->encoding) + deflateEnd(&c->gzip); + else + inflateEnd(&c->gzip); +#if HAVE_BZIP2 + } else if (c->type == IMPORT_COMPRESS_BZIP2) { + if (c->encoding) + BZ2_bzCompressEnd(&c->bzip2); + else + BZ2_bzDecompressEnd(&c->bzip2); +#endif + } + + c->type = IMPORT_COMPRESS_UNKNOWN; +} + +int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) { + static const uint8_t xz_signature[] = { + 0xfd, '7', 'z', 'X', 'Z', 0x00 + }; + static const uint8_t gzip_signature[] = { + 0x1f, 0x8b + }; + static const uint8_t bzip2_signature[] = { + 'B', 'Z', 'h' + }; + + int r; + + assert(c); + + if (c->type != IMPORT_COMPRESS_UNKNOWN) + return 1; + + if (size < MAX3(sizeof(xz_signature), + sizeof(gzip_signature), + sizeof(bzip2_signature))) + return 0; + + assert(data); + + if (memcmp(data, xz_signature, sizeof(xz_signature)) == 0) { + lzma_ret xzr; + + xzr = lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED); + if (xzr != LZMA_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_XZ; + + } else if (memcmp(data, gzip_signature, sizeof(gzip_signature)) == 0) { + r = inflateInit2(&c->gzip, 15+16); + if (r != Z_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_GZIP; + +#if HAVE_BZIP2 + } else if (memcmp(data, bzip2_signature, sizeof(bzip2_signature)) == 0) { + r = BZ2_bzDecompressInit(&c->bzip2, 0, 0); + if (r != BZ_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_BZIP2; +#endif + } else + c->type = IMPORT_COMPRESS_UNCOMPRESSED; + + c->encoding = false; + + return 1; +} + +int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata) { + int r; + + assert(c); + assert(callback); + + r = import_uncompress_detect(c, data, size); + if (r <= 0) + return r; + + if (c->encoding) + return -EINVAL; + + if (size <= 0) + return 1; + + assert(data); + + switch (c->type) { + + case IMPORT_COMPRESS_UNCOMPRESSED: + r = callback(data, size, userdata); + if (r < 0) + return r; + + break; + + case IMPORT_COMPRESS_XZ: + c->xz.next_in = data; + c->xz.avail_in = size; + + while (c->xz.avail_in > 0) { + uint8_t buffer[16 * 1024]; + lzma_ret lzr; + + c->xz.next_out = buffer; + c->xz.avail_out = sizeof(buffer); + + lzr = lzma_code(&c->xz, LZMA_RUN); + if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) + return -EIO; + + r = callback(buffer, sizeof(buffer) - c->xz.avail_out, userdata); + if (r < 0) + return r; + } + + break; + + case IMPORT_COMPRESS_GZIP: + c->gzip.next_in = (void*) data; + c->gzip.avail_in = size; + + while (c->gzip.avail_in > 0) { + uint8_t buffer[16 * 1024]; + + c->gzip.next_out = buffer; + c->gzip.avail_out = sizeof(buffer); + + r = inflate(&c->gzip, Z_NO_FLUSH); + if (!IN_SET(r, Z_OK, Z_STREAM_END)) + return -EIO; + + r = callback(buffer, sizeof(buffer) - c->gzip.avail_out, userdata); + if (r < 0) + return r; + } + + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + c->bzip2.next_in = (void*) data; + c->bzip2.avail_in = size; + + while (c->bzip2.avail_in > 0) { + uint8_t buffer[16 * 1024]; + + c->bzip2.next_out = (char*) buffer; + c->bzip2.avail_out = sizeof(buffer); + + r = BZ2_bzDecompress(&c->bzip2); + if (!IN_SET(r, BZ_OK, BZ_STREAM_END)) + return -EIO; + + r = callback(buffer, sizeof(buffer) - c->bzip2.avail_out, userdata); + if (r < 0) + return r; + } + + break; +#endif + + default: + assert_not_reached("Unknown compression"); + } + + return 1; +} + +int import_compress_init(ImportCompress *c, ImportCompressType t) { + int r; + + assert(c); + + switch (t) { + + case IMPORT_COMPRESS_XZ: { + lzma_ret xzr; + + xzr = lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); + if (xzr != LZMA_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_XZ; + break; + } + + case IMPORT_COMPRESS_GZIP: + r = deflateInit2(&c->gzip, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY); + if (r != Z_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_GZIP; + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + r = BZ2_bzCompressInit(&c->bzip2, 9, 0, 0); + if (r != BZ_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_BZIP2; + break; +#endif + + case IMPORT_COMPRESS_UNCOMPRESSED: + c->type = IMPORT_COMPRESS_UNCOMPRESSED; + break; + + default: + return -EOPNOTSUPP; + } + + c->encoding = true; + return 0; +} + +static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + size_t l; + void *p; + + if (*buffer_allocated > *buffer_size) + return 0; + + l = MAX(16*1024U, (*buffer_size * 2)); + p = realloc(*buffer, l); + if (!p) + return -ENOMEM; + + *buffer = p; + *buffer_allocated = l; + + return 1; +} + +int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + int r; + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + if (size <= 0) + return 0; + + assert(data); + + *buffer_size = 0; + + switch (c->type) { + + case IMPORT_COMPRESS_XZ: + + c->xz.next_in = data; + c->xz.avail_in = size; + + while (c->xz.avail_in > 0) { + lzma_ret lzr; + + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = lzma_code(&c->xz, LZMA_RUN); + if (lzr != LZMA_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } + + break; + + case IMPORT_COMPRESS_GZIP: + + c->gzip.next_in = (void*) data; + c->gzip.avail_in = size; + + while (c->gzip.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = *buffer_allocated - *buffer_size; + + r = deflate(&c->gzip, Z_NO_FLUSH); + if (r != Z_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; + } + + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + + c->bzip2.next_in = (void*) data; + c->bzip2.avail_in = size; + + while (c->bzip2.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = *buffer_allocated - *buffer_size; + + r = BZ2_bzCompress(&c->bzip2, BZ_RUN); + if (r != BZ_RUN_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; + } + + break; +#endif + + case IMPORT_COMPRESS_UNCOMPRESSED: + + if (*buffer_allocated < size) { + void *p; + + p = realloc(*buffer, size); + if (!p) + return -ENOMEM; + + *buffer = p; + *buffer_allocated = size; + } + + memcpy(*buffer, data, size); + *buffer_size = size; + break; + + default: + return -EOPNOTSUPP; + } + + return 0; +} + +int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + int r; + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + *buffer_size = 0; + + switch (c->type) { + + case IMPORT_COMPRESS_XZ: { + lzma_ret lzr; + + c->xz.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = lzma_code(&c->xz, LZMA_FINISH); + if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } while (lzr != LZMA_STREAM_END); + + break; + } + + case IMPORT_COMPRESS_GZIP: + c->gzip.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = *buffer_allocated - *buffer_size; + + r = deflate(&c->gzip, Z_FINISH); + if (!IN_SET(r, Z_OK, Z_STREAM_END)) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; + } while (r != Z_STREAM_END); + + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + c->bzip2.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = *buffer_allocated - *buffer_size; + + r = BZ2_bzCompress(&c->bzip2, BZ_FINISH); + if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END)) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; + } while (r != BZ_STREAM_END); + + break; +#endif + + case IMPORT_COMPRESS_UNCOMPRESSED: + break; + + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static const char* const import_compress_type_table[_IMPORT_COMPRESS_TYPE_MAX] = { + [IMPORT_COMPRESS_UNKNOWN] = "unknown", + [IMPORT_COMPRESS_UNCOMPRESSED] = "uncompressed", + [IMPORT_COMPRESS_XZ] = "xz", + [IMPORT_COMPRESS_GZIP] = "gzip", +#if HAVE_BZIP2 + [IMPORT_COMPRESS_BZIP2] = "bzip2", +#endif +}; + +DEFINE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType); diff --git a/src/import/import-compress.h b/src/import/import-compress.h new file mode 100644 index 0000000..859bd0e --- /dev/null +++ b/src/import/import-compress.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#if HAVE_BZIP2 +#include <bzlib.h> +#endif +#include <lzma.h> +#include <sys/types.h> +#include <zlib.h> + +#include "macro.h" + +typedef enum ImportCompressType { + IMPORT_COMPRESS_UNKNOWN, + IMPORT_COMPRESS_UNCOMPRESSED, + IMPORT_COMPRESS_XZ, + IMPORT_COMPRESS_GZIP, + IMPORT_COMPRESS_BZIP2, + _IMPORT_COMPRESS_TYPE_MAX, + _IMPORT_COMPRESS_TYPE_INVALID = -1, +} ImportCompressType; + +typedef struct ImportCompress { + ImportCompressType type; + bool encoding; + union { + lzma_stream xz; + z_stream gzip; +#if HAVE_BZIP2 + bz_stream bzip2; +#endif + }; +} ImportCompress; + +typedef int (*ImportCompressCallback)(const void *data, size_t size, void *userdata); + +void import_compress_free(ImportCompress *c); + +int import_uncompress_detect(ImportCompress *c, const void *data, size_t size); +int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata); + +int import_compress_init(ImportCompress *c, ImportCompressType t); +int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated); +int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated); + +const char* import_compress_type_to_string(ImportCompressType t) _const_; +ImportCompressType import_compress_type_from_string(const char *s) _pure_; diff --git a/src/import/import-fs.c b/src/import/import-fs.c new file mode 100644 index 0000000..35ba6ba --- /dev/null +++ b/src/import/import-fs.c @@ -0,0 +1,327 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <getopt.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-util.h" +#include "machine-image.h" +#include "mkdir.h" +#include "parse-util.h" +#include "ratelimit.h" +#include "rm-rf.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "verbs.h" + +static bool arg_force = false; +static bool arg_read_only = false; +static const char *arg_image_root = "/var/lib/machines"; + +typedef struct ProgressInfo { + RateLimit limit; + char *path; + uint64_t size; + bool started; + bool logged_incomplete; +} ProgressInfo; + +static volatile sig_atomic_t cancelled = false; + +static void sigterm_sigint(int sig) { + cancelled = true; +} + +static void progress_info_free(ProgressInfo *p) { + free(p->path); +} + +static void progress_show(ProgressInfo *p) { + assert(p); + + /* Show progress only every now and then. */ + if (!ratelimit_below(&p->limit)) + return; + + /* Suppress the first message, start with the second one */ + if (!p->started) { + p->started = true; + return; + } + + /* Mention the list is incomplete before showing first output. */ + if (!p->logged_incomplete) { + log_notice("(Note, file list shown below is incomplete, and is intended as sporadic progress report only.)"); + p->logged_incomplete = true; + } + + if (p->size == 0) + log_info("Copying tree, currently at '%s'...", p->path); + else { + char buffer[FORMAT_BYTES_MAX]; + + log_info("Copying tree, currently at '%s' (@%s)...", p->path, format_bytes(buffer, sizeof(buffer), p->size)); + } +} + +static int progress_path(const char *path, const struct stat *st, void *userdata) { + ProgressInfo *p = userdata; + int r; + + assert(p); + + if (cancelled) + return -EOWNERDEAD; + + r = free_and_strdup(&p->path, path); + if (r < 0) + return r; + + p->size = 0; + + progress_show(p); + return 0; +} + +static int progress_bytes(uint64_t nbytes, void *userdata) { + ProgressInfo *p = userdata; + + assert(p); + assert(p->size != UINT64_MAX); + + if (cancelled) + return -EOWNERDEAD; + + p->size += nbytes; + + progress_show(p); + return 0; +} + +static int import_fs(int argc, char *argv[], void *userdata) { + _cleanup_(rm_rf_subvolume_and_freep) char *temp_path = NULL; + _cleanup_(progress_info_free) ProgressInfo progress = {}; + const char *path = NULL, *local = NULL, *final_path; + _cleanup_close_ int open_fd = -1; + struct sigaction old_sigint_sa, old_sigterm_sa; + static const struct sigaction sa = { + .sa_handler = sigterm_sigint, + .sa_flags = SA_RESTART, + }; + int r, fd; + + if (argc >= 2) + path = argv[1]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (argc >= 3) + local = argv[2]; + else if (path) + local = basename(path); + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (local) { + if (!machine_name_is_valid(local)) { + log_error("Local image name '%s' is not valid.", local); + return -EINVAL; + } + + if (!arg_force) { + r = image_find(IMAGE_MACHINE, local, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else { + log_error("Image '%s' already exists.", local); + return -EEXIST; + } + } + } else + local = "imported"; + + if (path) { + open_fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open directory to import: %m"); + + fd = open_fd; + + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDIN_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); + } + + final_path = strjoina(arg_image_root, "/", local); + + r = tempfn_random(final_path, NULL, &temp_path); + if (r < 0) + return log_oom(); + + (void) mkdir_parents_label(temp_path, 0700); + + RATELIMIT_INIT(progress.limit, 200*USEC_PER_MSEC, 1); + + /* Hook into SIGINT/SIGTERM, so that we can cancel things then */ + assert(sigaction(SIGINT, &sa, &old_sigint_sa) >= 0); + assert(sigaction(SIGTERM, &sa, &old_sigterm_sa) >= 0); + + r = btrfs_subvol_snapshot_fd_full( + fd, + temp_path, + BTRFS_SNAPSHOT_FALLBACK_COPY|BTRFS_SNAPSHOT_RECURSIVE|BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|BTRFS_SNAPSHOT_QUOTA, + progress_path, + progress_bytes, + &progress); + if (r == -EOWNERDEAD) { /* SIGINT + SIGTERM cause this, see signal handler above */ + log_error("Copy cancelled."); + goto finish; + } + if (r < 0) { + log_error_errno(r, "Failed to copy directory: %m"); + goto finish; + } + + r = import_mangle_os_tree(temp_path); + if (r < 0) + goto finish; + + (void) import_assign_pool_quota_and_warn(temp_path); + + if (arg_read_only) { + r = import_make_read_only(temp_path); + if (r < 0) { + log_error_errno(r, "Failed to make directory read-only: %m"); + goto finish; + } + } + + if (arg_force) + (void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + r = rename_noreplace(AT_FDCWD, temp_path, AT_FDCWD, final_path); + if (r < 0) { + log_error_errno(r, "Failed to move image into place: %m"); + goto finish; + } + + temp_path = mfree(temp_path); + + log_info("Exiting."); + +finish: + /* Put old signal handlers into place */ + assert(sigaction(SIGINT, &old_sigint_sa, NULL) >= 0); + assert(sigaction(SIGTERM, &old_sigterm_sa, NULL) >= 0); + + return 0; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%s [OPTIONS...] {COMMAND} ...\n\n" + "Import container images from a file system.\n\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --force Force creation of image\n" + " --image-root=PATH Image root directory\n" + " --read-only Create a read-only image\n\n" + "Commands:\n" + " run DIRECTORY [NAME] Import a directory\n", + program_invocation_short_name); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORCE, + ARG_IMAGE_ROOT, + ARG_READ_ONLY, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "force", no_argument, NULL, ARG_FORCE }, + { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + {} + }; + + int c; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORCE: + arg_force = true; + break; + + case ARG_IMAGE_ROOT: + arg_image_root = optarg; + break; + + case ARG_READ_ONLY: + arg_read_only = true; + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached("Unhandled option"); + } + + return 1; +} + +static int import_fs_main(int argc, char *argv[]) { + + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "run", 2, 3, 0, import_fs }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +int main(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + goto finish; + + r = import_fs_main(argc, argv); + +finish: + return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/import/import-pubring.gpg b/src/import/import-pubring.gpg Binary files differnew file mode 100644 index 0000000..be27776 --- /dev/null +++ b/src/import/import-pubring.gpg diff --git a/src/import/import-raw.c b/src/import/import-raw.c new file mode 100644 index 0000000..4b11615 --- /dev/null +++ b/src/import/import-raw.c @@ -0,0 +1,438 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <linux/fs.h> + +#include "sd-daemon.h" +#include "sd-event.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "chattr-util.h" +#include "copy.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-compress.h" +#include "import-raw.h" +#include "io-util.h" +#include "machine-pool.h" +#include "mkdir.h" +#include "path-util.h" +#include "qcow2-util.h" +#include "ratelimit.h" +#include "rm-rf.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +struct RawImport { + sd_event *event; + + char *image_root; + + RawImportFinished on_finished; + void *userdata; + + char *local; + bool force_local; + bool read_only; + + char *temp_path; + char *final_path; + + int input_fd; + int output_fd; + + ImportCompress compress; + + sd_event_source *input_event_source; + + uint8_t buffer[16*1024]; + size_t buffer_size; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + struct stat st; + + unsigned last_percent; + RateLimit progress_rate_limit; +}; + +RawImport* raw_import_unref(RawImport *i) { + if (!i) + return NULL; + + sd_event_unref(i->event); + + if (i->temp_path) { + (void) unlink(i->temp_path); + free(i->temp_path); + } + + import_compress_free(&i->compress); + + sd_event_source_unref(i->input_event_source); + + safe_close(i->output_fd); + + free(i->final_path); + free(i->image_root); + free(i->local); + return mfree(i); +} + +int raw_import_new( + RawImport **ret, + sd_event *event, + const char *image_root, + RawImportFinished on_finished, + void *userdata) { + + _cleanup_(raw_import_unrefp) RawImport *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + i = new(RawImport, 1); + if (!i) + return -ENOMEM; + + *i = (RawImport) { + .input_fd = -1, + .output_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .last_percent = (unsigned) -1, + .image_root = TAKE_PTR(root), + }; + + RATELIMIT_INIT(i->progress_rate_limit, 100 * USEC_PER_MSEC, 1); + + if (event) + i->event = sd_event_ref(event); + else { + r = sd_event_default(&i->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(i); + + return 0; +} + +static void raw_import_report_progress(RawImport *i) { + unsigned percent; + assert(i); + + /* We have no size information, unless the source is a regular file */ + if (!S_ISREG(i->st.st_mode)) + return; + + if (i->written_compressed >= (uint64_t) i->st.st_size) + percent = 100; + else + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); + + if (percent == i->last_percent) + return; + + if (!ratelimit_below(&i->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Imported %u%%.", percent); + + i->last_percent = percent; +} + +static int raw_import_maybe_convert_qcow2(RawImport *i) { + _cleanup_close_ int converted_fd = -1; + _cleanup_free_ char *t = NULL; + int r; + + assert(i); + + r = qcow2_detect(i->output_fd); + if (r < 0) + return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); + if (r == 0) + return 0; + + /* This is a QCOW2 image, let's convert it */ + r = tempfn_random(i->final_path, NULL, &t); + if (r < 0) + return log_oom(); + + converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (converted_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", t); + + r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL, NULL); + if (r < 0) + log_warning_errno(r, "Failed to set file attributes on %s: %m", t); + + log_info("Unpacking QCOW2 file."); + + r = qcow2_convert(i->output_fd, converted_fd); + if (r < 0) { + unlink(t); + return log_error_errno(r, "Failed to convert qcow2 image: %m"); + } + + (void) unlink(i->temp_path); + free_and_replace(i->temp_path, t); + + safe_close(i->output_fd); + i->output_fd = TAKE_FD(converted_fd); + + return 1; +} + +static int raw_import_finish(RawImport *i) { + int r; + + assert(i); + assert(i->output_fd >= 0); + assert(i->temp_path); + assert(i->final_path); + + /* In case this was a sparse file, make sure the file system is right */ + if (i->written_uncompressed > 0) { + if (ftruncate(i->output_fd, i->written_uncompressed) < 0) + return log_error_errno(errno, "Failed to truncate file: %m"); + } + + r = raw_import_maybe_convert_qcow2(i); + if (r < 0) + return r; + + if (S_ISREG(i->st.st_mode)) { + (void) copy_times(i->input_fd, i->output_fd); + (void) copy_xattr(i->input_fd, i->output_fd); + } + + if (i->read_only) { + r = import_make_read_only_fd(i->output_fd); + if (r < 0) + return r; + } + + if (i->force_local) + (void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + if (r < 0) + return log_error_errno(r, "Failed to move image into place: %m"); + + i->temp_path = mfree(i->temp_path); + + return 0; +} + +static int raw_import_open_disk(RawImport *i) { + int r; + + assert(i); + + assert(!i->final_path); + assert(!i->temp_path); + assert(i->output_fd < 0); + + i->final_path = strjoin(i->image_root, "/", i->local, ".raw"); + if (!i->final_path) + return log_oom(); + + r = tempfn_random(i->final_path, NULL, &i->temp_path); + if (r < 0) + return log_oom(); + + (void) mkdir_parents_label(i->temp_path, 0700); + + i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (i->output_fd < 0) + return log_error_errno(errno, "Failed to open destination %s: %m", i->temp_path); + + r = chattr_fd(i->output_fd, FS_NOCOW_FL, FS_NOCOW_FL, NULL); + if (r < 0) + log_warning_errno(r, "Failed to set file attributes on %s: %m", i->temp_path); + + return 0; +} + +static int raw_import_try_reflink(RawImport *i) { + off_t p; + int r; + + assert(i); + assert(i->input_fd >= 0); + assert(i->output_fd >= 0); + + if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED) + return 0; + + if (!S_ISREG(i->st.st_mode)) + return 0; + + p = lseek(i->input_fd, 0, SEEK_CUR); + if (p == (off_t) -1) + return log_error_errno(errno, "Failed to read file offset of input file: %m"); + + /* Let's only try a btrfs reflink, if we are reading from the beginning of the file */ + if ((uint64_t) p != (uint64_t) i->buffer_size) + return 0; + + r = btrfs_reflink(i->input_fd, i->output_fd); + if (r >= 0) + return 1; + + return 0; +} + +static int raw_import_write(const void *p, size_t sz, void *userdata) { + RawImport *i = userdata; + ssize_t n; + + n = sparse_write(i->output_fd, p, sz, 64); + if (n < 0) + return (int) n; + if ((size_t) n < sz) + return -EIO; + + i->written_uncompressed += sz; + + return 0; +} + +static int raw_import_process(RawImport *i) { + ssize_t l; + int r; + + assert(i); + assert(i->buffer_size < sizeof(i->buffer)); + + l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to read input file: %m"); + goto finish; + } + if (l == 0) { + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + log_error("Premature end of file."); + r = -EIO; + goto finish; + } + + r = raw_import_finish(i); + goto finish; + } + + i->buffer_size += l; + + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + if (r < 0) { + log_error_errno(r, "Failed to detect file compression: %m"); + goto finish; + } + if (r == 0) /* Need more data */ + return 0; + + r = raw_import_open_disk(i); + if (r < 0) + goto finish; + + r = raw_import_try_reflink(i); + if (r < 0) + goto finish; + if (r > 0) { + r = raw_import_finish(i); + goto finish; + } + } + + r = import_uncompress(&i->compress, i->buffer, i->buffer_size, raw_import_write, i); + if (r < 0) { + log_error_errno(r, "Failed to decode and write: %m"); + goto finish; + } + + i->written_compressed += i->buffer_size; + i->buffer_size = 0; + + raw_import_report_progress(i); + + return 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); + + return 0; +} + +static int raw_import_on_input(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + RawImport *i = userdata; + + return raw_import_process(i); +} + +static int raw_import_on_defer(sd_event_source *s, void *userdata) { + RawImport *i = userdata; + + return raw_import_process(i); +} + +int raw_import_start(RawImport *i, int fd, const char *local, bool force_local, bool read_only) { + int r; + + assert(i); + assert(fd >= 0); + assert(local); + + if (!machine_name_is_valid(local)) + return -EINVAL; + + if (i->input_fd >= 0) + return -EBUSY; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + i->force_local = force_local; + i->read_only = read_only; + + if (fstat(fd, &i->st) < 0) + return -errno; + + r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i); + if (r == -EPERM) { + /* This fd does not support epoll, for example because it is a regular file. Busy read in that case */ + r = sd_event_add_defer(i->event, &i->input_event_source, raw_import_on_defer, i); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(i->input_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + i->input_fd = fd; + return r; +} diff --git a/src/import/import-raw.h b/src/import/import-raw.h new file mode 100644 index 0000000..de4c3ea --- /dev/null +++ b/src/import/import-raw.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include "sd-event.h" + +#include "import-util.h" +#include "macro.h" + +typedef struct RawImport RawImport; + +typedef void (*RawImportFinished)(RawImport *import, int error, void *userdata); + +int raw_import_new(RawImport **import, sd_event *event, const char *image_root, RawImportFinished on_finished, void *userdata); +RawImport* raw_import_unref(RawImport *import); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref); + +int raw_import_start(RawImport *i, int fd, const char *local, bool force_local, bool read_only); diff --git a/src/import/import-tar.c b/src/import/import-tar.c new file mode 100644 index 0000000..4d0d925 --- /dev/null +++ b/src/import/import-tar.c @@ -0,0 +1,369 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <linux/fs.h> + +#include "sd-daemon.h" +#include "sd-event.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "fd-util.h" +#include "fileio.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-compress.h" +#include "import-tar.h" +#include "io-util.h" +#include "machine-pool.h" +#include "mkdir.h" +#include "path-util.h" +#include "process-util.h" +#include "qcow2-util.h" +#include "ratelimit.h" +#include "rm-rf.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +struct TarImport { + sd_event *event; + + char *image_root; + + TarImportFinished on_finished; + void *userdata; + + char *local; + bool force_local; + bool read_only; + + char *temp_path; + char *final_path; + + int input_fd; + int tar_fd; + + ImportCompress compress; + + sd_event_source *input_event_source; + + uint8_t buffer[16*1024]; + size_t buffer_size; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + struct stat st; + + pid_t tar_pid; + + unsigned last_percent; + RateLimit progress_rate_limit; +}; + +TarImport* tar_import_unref(TarImport *i) { + if (!i) + return NULL; + + sd_event_source_unref(i->input_event_source); + + if (i->tar_pid > 1) { + (void) kill_and_sigcont(i->tar_pid, SIGKILL); + (void) wait_for_terminate(i->tar_pid, NULL); + } + + if (i->temp_path) { + (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + free(i->temp_path); + } + + import_compress_free(&i->compress); + + sd_event_unref(i->event); + + safe_close(i->tar_fd); + + free(i->final_path); + free(i->image_root); + free(i->local); + return mfree(i); +} + +int tar_import_new( + TarImport **ret, + sd_event *event, + const char *image_root, + TarImportFinished on_finished, + void *userdata) { + + _cleanup_(tar_import_unrefp) TarImport *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + i = new(TarImport, 1); + if (!i) + return -ENOMEM; + + *i = (TarImport) { + .input_fd = -1, + .tar_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .last_percent = (unsigned) -1, + .image_root = TAKE_PTR(root), + }; + + RATELIMIT_INIT(i->progress_rate_limit, 100 * USEC_PER_MSEC, 1); + + if (event) + i->event = sd_event_ref(event); + else { + r = sd_event_default(&i->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(i); + + return 0; +} + +static void tar_import_report_progress(TarImport *i) { + unsigned percent; + assert(i); + + /* We have no size information, unless the source is a regular file */ + if (!S_ISREG(i->st.st_mode)) + return; + + if (i->written_compressed >= (uint64_t) i->st.st_size) + percent = 100; + else + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); + + if (percent == i->last_percent) + return; + + if (!ratelimit_below(&i->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Imported %u%%.", percent); + + i->last_percent = percent; +} + +static int tar_import_finish(TarImport *i) { + int r; + + assert(i); + assert(i->tar_fd >= 0); + assert(i->temp_path); + assert(i->final_path); + + i->tar_fd = safe_close(i->tar_fd); + + if (i->tar_pid > 0) { + r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG); + i->tar_pid = 0; + if (r < 0) + return r; + if (r != EXIT_SUCCESS) + return -EPROTO; + } + + r = import_mangle_os_tree(i->temp_path); + if (r < 0) + return r; + + if (i->read_only) { + r = import_make_read_only(i->temp_path); + if (r < 0) + return r; + } + + if (i->force_local) + (void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + if (r < 0) + return log_error_errno(r, "Failed to move image into place: %m"); + + i->temp_path = mfree(i->temp_path); + + return 0; +} + +static int tar_import_fork_tar(TarImport *i) { + int r; + + assert(i); + + assert(!i->final_path); + assert(!i->temp_path); + assert(i->tar_fd < 0); + + i->final_path = strjoin(i->image_root, "/", i->local); + if (!i->final_path) + return log_oom(); + + r = tempfn_random(i->final_path, NULL, &i->temp_path); + if (r < 0) + return log_oom(); + + (void) mkdir_parents_label(i->temp_path, 0700); + + r = btrfs_subvol_make(i->temp_path); + if (r == -ENOTTY) { + if (mkdir(i->temp_path, 0755) < 0) + return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path); + } else if (r < 0) + return log_error_errno(r, "Failed to create subvolume %s: %m", i->temp_path); + else + (void) import_assign_pool_quota_and_warn(i->temp_path); + + i->tar_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); + if (i->tar_fd < 0) + return i->tar_fd; + + return 0; +} + +static int tar_import_write(const void *p, size_t sz, void *userdata) { + TarImport *i = userdata; + int r; + + r = loop_write(i->tar_fd, p, sz, false); + if (r < 0) + return r; + + i->written_uncompressed += sz; + + return 0; +} + +static int tar_import_process(TarImport *i) { + ssize_t l; + int r; + + assert(i); + assert(i->buffer_size < sizeof(i->buffer)); + + l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to read input file: %m"); + goto finish; + } + if (l == 0) { + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + log_error("Premature end of file."); + r = -EIO; + goto finish; + } + + r = tar_import_finish(i); + goto finish; + } + + i->buffer_size += l; + + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + if (r < 0) { + log_error_errno(r, "Failed to detect file compression: %m"); + goto finish; + } + if (r == 0) /* Need more data */ + return 0; + + r = tar_import_fork_tar(i); + if (r < 0) + goto finish; + } + + r = import_uncompress(&i->compress, i->buffer, i->buffer_size, tar_import_write, i); + if (r < 0) { + log_error_errno(r, "Failed to decode and write: %m"); + goto finish; + } + + i->written_compressed += i->buffer_size; + i->buffer_size = 0; + + tar_import_report_progress(i); + + return 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); + + return 0; +} + +static int tar_import_on_input(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + TarImport *i = userdata; + + return tar_import_process(i); +} + +static int tar_import_on_defer(sd_event_source *s, void *userdata) { + TarImport *i = userdata; + + return tar_import_process(i); +} + +int tar_import_start(TarImport *i, int fd, const char *local, bool force_local, bool read_only) { + int r; + + assert(i); + assert(fd >= 0); + assert(local); + + if (!machine_name_is_valid(local)) + return -EINVAL; + + if (i->input_fd >= 0) + return -EBUSY; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + i->force_local = force_local; + i->read_only = read_only; + + if (fstat(fd, &i->st) < 0) + return -errno; + + r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i); + if (r == -EPERM) { + /* This fd does not support epoll, for example because it is a regular file. Busy read in that case */ + r = sd_event_add_defer(i->event, &i->input_event_source, tar_import_on_defer, i); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(i->input_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + i->input_fd = fd; + return r; +} diff --git a/src/import/import-tar.h b/src/import/import-tar.h new file mode 100644 index 0000000..347f522 --- /dev/null +++ b/src/import/import-tar.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include "sd-event.h" + +#include "import-util.h" +#include "macro.h" + +typedef struct TarImport TarImport; + +typedef void (*TarImportFinished)(TarImport *import, int error, void *userdata); + +int tar_import_new(TarImport **import, sd_event *event, const char *image_root, TarImportFinished on_finished, void *userdata); +TarImport* tar_import_unref(TarImport *import); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarImport*, tar_import_unref); + +int tar_import_start(TarImport *import, int fd, const char *local, bool force_local, bool read_only); diff --git a/src/import/import.c b/src/import/import.c new file mode 100644 index 0000000..f34244a --- /dev/null +++ b/src/import/import.c @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <getopt.h> + +#include "sd-event.h" +#include "sd-id128.h" + +#include "alloc-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-raw.h" +#include "import-tar.h" +#include "import-util.h" +#include "machine-image.h" +#include "main-func.h" +#include "signal-util.h" +#include "string-util.h" +#include "verbs.h" + +static bool arg_force = false; +static bool arg_read_only = false; +static const char *arg_image_root = "/var/lib/machines"; + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +static void on_tar_finished(TarImport *import, int error, void *userdata) { + sd_event *event = userdata; + assert(import); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int import_tar(int argc, char *argv[], void *userdata) { + _cleanup_(tar_import_unrefp) TarImport *import = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + const char *path = NULL, *local = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (argc >= 2) + path = argv[1]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (argc >= 3) + local = argv[2]; + else if (path) + local = basename(path); + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (local) { + r = tar_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local image name '%s' is not valid.", local); + return -EINVAL; + } + + if (!arg_force) { + r = image_find(IMAGE_MACHINE, local, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else { + log_error("Image '%s' already exists.", local); + return -EEXIST; + } + } + } else + local = "imported"; + + if (path) { + open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open tar image to import: %m"); + + fd = open_fd; + + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDIN_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Importing '%s', saving as '%s'.", strna(pretty), local); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate importer: %m"); + + r = tar_import_start(import, fd, local, arg_force, arg_read_only); + if (r < 0) + return log_error_errno(r, "Failed to import image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawImport *import, int error, void *userdata) { + sd_event *event = userdata; + assert(import); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int import_raw(int argc, char *argv[], void *userdata) { + _cleanup_(raw_import_unrefp) RawImport *import = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + const char *path = NULL, *local = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (argc >= 2) + path = argv[1]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (argc >= 3) + local = argv[2]; + else if (path) + local = basename(path); + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (local) { + r = raw_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local image name '%s' is not valid.", local); + return -EINVAL; + } + + if (!arg_force) { + r = image_find(IMAGE_MACHINE, local, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else { + log_error("Image '%s' already exists.", local); + return -EEXIST; + } + } + } else + local = "imported"; + + if (path) { + open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open raw image to import: %m"); + + fd = open_fd; + + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDIN_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate importer: %m"); + + r = raw_import_start(import, fd, local, arg_force, arg_read_only); + if (r < 0) + return log_error_errno(r, "Failed to import image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%s [OPTIONS...] {COMMAND} ...\n\n" + "Import container or virtual machine images.\n\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --force Force creation of image\n" + " --image-root=PATH Image root directory\n" + " --read-only Create a read-only image\n\n" + "Commands:\n" + " tar FILE [NAME] Import a TAR image\n" + " raw FILE [NAME] Import a RAW image\n", + program_invocation_short_name); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORCE, + ARG_IMAGE_ROOT, + ARG_READ_ONLY, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "force", no_argument, NULL, ARG_FORCE }, + { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + {} + }; + + int c; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORCE: + arg_force = true; + break; + + case ARG_IMAGE_ROOT: + arg_image_root = optarg; + break; + + case ARG_READ_ONLY: + arg_read_only = true; + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached("Unhandled option"); + } + + return 1; +} + +static int import_main(int argc, char *argv[]) { + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, import_tar }, + { "raw", 2, 3, 0, import_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +static int run(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + return 0; + + (void) ignore_signals(SIGPIPE, -1); + + return import_main(argc, argv); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/importd.c b/src/import/importd.c new file mode 100644 index 0000000..2426933 --- /dev/null +++ b/src/import/importd.c @@ -0,0 +1,1266 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <sys/prctl.h> +#include <sys/wait.h> + +#include "sd-bus.h" + +#include "alloc-util.h" +#include "bus-common-errors.h" +#include "bus-util.h" +#include "def.h" +#include "fd-util.h" +#include "float.h" +#include "hostname-util.h" +#include "import-util.h" +#include "machine-pool.h" +#include "main-func.h" +#include "missing.h" +#include "mkdir.h" +#include "parse-util.h" +#include "path-util.h" +#include "process-util.h" +#include "signal-util.h" +#include "socket-util.h" +#include "stat-util.h" +#include "string-table.h" +#include "strv.h" +#include "syslog-util.h" +#include "user-util.h" +#include "util.h" +#include "web-util.h" + +typedef struct Transfer Transfer; +typedef struct Manager Manager; + +typedef enum TransferType { + TRANSFER_IMPORT_TAR, + TRANSFER_IMPORT_RAW, + TRANSFER_IMPORT_FS, + TRANSFER_EXPORT_TAR, + TRANSFER_EXPORT_RAW, + TRANSFER_PULL_TAR, + TRANSFER_PULL_RAW, + _TRANSFER_TYPE_MAX, + _TRANSFER_TYPE_INVALID = -1, +} TransferType; + +struct Transfer { + Manager *manager; + + uint32_t id; + char *object_path; + + TransferType type; + ImportVerify verify; + + char *remote; + char *local; + bool force_local; + bool read_only; + + char *format; + + pid_t pid; + + int log_fd; + + char log_message[LINE_MAX]; + size_t log_message_size; + + sd_event_source *pid_event_source; + sd_event_source *log_event_source; + + unsigned n_canceled; + unsigned progress_percent; + + int stdin_fd; + int stdout_fd; +}; + +struct Manager { + sd_event *event; + sd_bus *bus; + + uint32_t current_transfer_id; + Hashmap *transfers; + + Hashmap *polkit_registry; + + int notify_fd; + + sd_event_source *notify_event_source; +}; + +#define TRANSFERS_MAX 64 + +static const char* const transfer_type_table[_TRANSFER_TYPE_MAX] = { + [TRANSFER_IMPORT_TAR] = "import-tar", + [TRANSFER_IMPORT_RAW] = "import-raw", + [TRANSFER_IMPORT_FS] = "import-fs", + [TRANSFER_EXPORT_TAR] = "export-tar", + [TRANSFER_EXPORT_RAW] = "export-raw", + [TRANSFER_PULL_TAR] = "pull-tar", + [TRANSFER_PULL_RAW] = "pull-raw", +}; + +DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(transfer_type, TransferType); + +static Transfer *transfer_unref(Transfer *t) { + if (!t) + return NULL; + + if (t->manager) + hashmap_remove(t->manager->transfers, UINT32_TO_PTR(t->id)); + + sd_event_source_unref(t->pid_event_source); + sd_event_source_unref(t->log_event_source); + + free(t->remote); + free(t->local); + free(t->format); + free(t->object_path); + + if (t->pid > 0) { + (void) kill_and_sigcont(t->pid, SIGKILL); + (void) wait_for_terminate(t->pid, NULL); + } + + safe_close(t->log_fd); + safe_close(t->stdin_fd); + safe_close(t->stdout_fd); + + return mfree(t); +} + +DEFINE_TRIVIAL_CLEANUP_FUNC(Transfer*, transfer_unref); + +static int transfer_new(Manager *m, Transfer **ret) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + uint32_t id; + int r; + + assert(m); + assert(ret); + + if (hashmap_size(m->transfers) >= TRANSFERS_MAX) + return -E2BIG; + + r = hashmap_ensure_allocated(&m->transfers, &trivial_hash_ops); + if (r < 0) + return r; + + t = new(Transfer, 1); + if (!t) + return -ENOMEM; + + *t = (Transfer) { + .type = _TRANSFER_TYPE_INVALID, + .log_fd = -1, + .stdin_fd = -1, + .stdout_fd = -1, + .verify = _IMPORT_VERIFY_INVALID, + .progress_percent= (unsigned) -1, + }; + + id = m->current_transfer_id + 1; + + if (asprintf(&t->object_path, "/org/freedesktop/import1/transfer/_%" PRIu32, id) < 0) + return -ENOMEM; + + r = hashmap_put(m->transfers, UINT32_TO_PTR(id), t); + if (r < 0) + return r; + + m->current_transfer_id = id; + + t->manager = m; + t->id = id; + + *ret = TAKE_PTR(t); + + return 0; +} + +static double transfer_percent_as_double(Transfer *t) { + assert(t); + + if (t->progress_percent == (unsigned) -1) + return -DBL_MAX; + + return (double) t->progress_percent / 100.0; +} + +static void transfer_send_log_line(Transfer *t, const char *line) { + int r, priority = LOG_INFO; + + assert(t); + assert(line); + + syslog_parse_priority(&line, &priority, true); + + log_full(priority, "(transfer%" PRIu32 ") %s", t->id, line); + + r = sd_bus_emit_signal( + t->manager->bus, + t->object_path, + "org.freedesktop.import1.Transfer", + "LogMessage", + "us", + priority, + line); + if (r < 0) + log_warning_errno(r, "Cannot emit log message signal, ignoring: %m"); + } + +static void transfer_send_logs(Transfer *t, bool flush) { + assert(t); + + /* Try to send out all log messages, if we can. But if we + * can't we remove the messages from the buffer, but don't + * fail */ + + while (t->log_message_size > 0) { + _cleanup_free_ char *n = NULL; + char *e; + + if (t->log_message_size >= sizeof(t->log_message)) + e = t->log_message + sizeof(t->log_message); + else { + char *a, *b; + + a = memchr(t->log_message, 0, t->log_message_size); + b = memchr(t->log_message, '\n', t->log_message_size); + + if (a && b) + e = a < b ? a : b; + else if (a) + e = a; + else + e = b; + } + + if (!e) { + if (!flush) + return; + + e = t->log_message + t->log_message_size; + } + + n = strndup(t->log_message, e - t->log_message); + + /* Skip over NUL and newlines */ + while (e < t->log_message + t->log_message_size && (*e == 0 || *e == '\n')) + e++; + + memmove(t->log_message, e, t->log_message + sizeof(t->log_message) - e); + t->log_message_size -= e - t->log_message; + + if (!n) { + log_oom(); + continue; + } + + if (isempty(n)) + continue; + + transfer_send_log_line(t, n); + } +} + +static int transfer_finalize(Transfer *t, bool success) { + int r; + + assert(t); + + transfer_send_logs(t, true); + + r = sd_bus_emit_signal( + t->manager->bus, + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "TransferRemoved", + "uos", + t->id, + t->object_path, + success ? "done" : + t->n_canceled > 0 ? "canceled" : "failed"); + + if (r < 0) + log_error_errno(r, "Cannot emit message: %m"); + + transfer_unref(t); + return 0; +} + +static int transfer_cancel(Transfer *t) { + int r; + + assert(t); + + r = kill_and_sigcont(t->pid, t->n_canceled < 3 ? SIGTERM : SIGKILL); + if (r < 0) + return r; + + t->n_canceled++; + return 0; +} + +static int transfer_on_pid(sd_event_source *s, const siginfo_t *si, void *userdata) { + Transfer *t = userdata; + bool success = false; + + assert(s); + assert(t); + + if (si->si_code == CLD_EXITED) { + if (si->si_status != 0) + log_error("Transfer process failed with exit code %i.", si->si_status); + else { + log_debug("Transfer process succeeded."); + success = true; + } + + } else if (IN_SET(si->si_code, CLD_KILLED, CLD_DUMPED)) + log_error("Transfer process terminated by signal %s.", signal_to_string(si->si_status)); + else + log_error("Transfer process failed due to unknown reason."); + + t->pid = 0; + + return transfer_finalize(t, success); +} + +static int transfer_on_log(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + Transfer *t = userdata; + ssize_t l; + + assert(s); + assert(t); + + l = read(fd, t->log_message + t->log_message_size, sizeof(t->log_message) - t->log_message_size); + if (l < 0) + log_error_errno(errno, "Failed to read log message: %m"); + if (l <= 0) { + /* EOF/read error. We just close the pipe here, and + * close the watch, waiting for the SIGCHLD to arrive, + * before we do anything else. */ + t->log_event_source = sd_event_source_unref(t->log_event_source); + return 0; + } + + t->log_message_size += l; + + transfer_send_logs(t, false); + + return 0; +} + +static int transfer_start(Transfer *t) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + int r; + + assert(t); + assert(t->pid <= 0); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return -errno; + + r = safe_fork("(sd-transfer)", FORK_RESET_SIGNALS|FORK_DEATHSIG, &t->pid); + if (r < 0) + return r; + if (r == 0) { + const char *cmd[] = { + NULL, /* systemd-import, systemd-import-fs, systemd-export or systemd-pull */ + NULL, /* tar, raw */ + NULL, /* --verify= */ + NULL, /* verify argument */ + NULL, /* maybe --force */ + NULL, /* maybe --read-only */ + NULL, /* if so: the actual URL */ + NULL, /* maybe --format= */ + NULL, /* if so: the actual format */ + NULL, /* remote */ + NULL, /* local */ + NULL + }; + unsigned k = 0; + + /* Child */ + + pipefd[0] = safe_close(pipefd[0]); + + r = rearrange_stdio(t->stdin_fd, + t->stdout_fd < 0 ? pipefd[1] : t->stdout_fd, + pipefd[1]); + if (r < 0) { + log_error_errno(r, "Failed to set stdin/stdout/stderr: %m"); + _exit(EXIT_FAILURE); + } + + if (setenv("SYSTEMD_LOG_TARGET", "console-prefixed", 1) < 0 || + setenv("NOTIFY_SOCKET", "/run/systemd/import/notify", 1) < 0) { + log_error_errno(errno, "setenv() failed: %m"); + _exit(EXIT_FAILURE); + } + + switch (t->type) { + + case TRANSFER_IMPORT_TAR: + case TRANSFER_IMPORT_RAW: + cmd[k++] = SYSTEMD_IMPORT_PATH; + break; + + case TRANSFER_IMPORT_FS: + cmd[k++] = SYSTEMD_IMPORT_FS_PATH; + break; + + case TRANSFER_EXPORT_TAR: + case TRANSFER_EXPORT_RAW: + cmd[k++] = SYSTEMD_EXPORT_PATH; + break; + + case TRANSFER_PULL_TAR: + case TRANSFER_PULL_RAW: + cmd[k++] = SYSTEMD_PULL_PATH; + break; + + default: + assert_not_reached("Unexpected transfer type"); + } + + switch (t->type) { + + case TRANSFER_IMPORT_TAR: + case TRANSFER_EXPORT_TAR: + case TRANSFER_PULL_TAR: + cmd[k++] = "tar"; + break; + + case TRANSFER_IMPORT_RAW: + case TRANSFER_EXPORT_RAW: + case TRANSFER_PULL_RAW: + cmd[k++] = "raw"; + break; + + case TRANSFER_IMPORT_FS: + cmd[k++] = "run"; + break; + + default: + break; + } + + if (t->verify != _IMPORT_VERIFY_INVALID) { + cmd[k++] = "--verify"; + cmd[k++] = import_verify_to_string(t->verify); + } + + if (t->force_local) + cmd[k++] = "--force"; + if (t->read_only) + cmd[k++] = "--read-only"; + + if (t->format) { + cmd[k++] = "--format"; + cmd[k++] = t->format; + } + + if (!IN_SET(t->type, TRANSFER_EXPORT_TAR, TRANSFER_EXPORT_RAW)) { + if (t->remote) + cmd[k++] = t->remote; + else + cmd[k++] = "-"; + } + + if (t->local) + cmd[k++] = t->local; + cmd[k] = NULL; + + execv(cmd[0], (char * const *) cmd); + log_error_errno(errno, "Failed to execute %s tool: %m", cmd[0]); + _exit(EXIT_FAILURE); + } + + pipefd[1] = safe_close(pipefd[1]); + t->log_fd = TAKE_FD(pipefd[0]); + + t->stdin_fd = safe_close(t->stdin_fd); + + r = sd_event_add_child(t->manager->event, &t->pid_event_source, t->pid, WEXITED, transfer_on_pid, t); + if (r < 0) + return r; + + r = sd_event_add_io(t->manager->event, &t->log_event_source, t->log_fd, EPOLLIN, transfer_on_log, t); + if (r < 0) + return r; + + /* Make sure always process logging before SIGCHLD */ + r = sd_event_source_set_priority(t->log_event_source, SD_EVENT_PRIORITY_NORMAL -5); + if (r < 0) + return r; + + r = sd_bus_emit_signal( + t->manager->bus, + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "TransferNew", + "uo", + t->id, + t->object_path); + if (r < 0) + return r; + + return 0; +} + +static Manager *manager_unref(Manager *m) { + Transfer *t; + + if (!m) + return NULL; + + sd_event_source_unref(m->notify_event_source); + safe_close(m->notify_fd); + + while ((t = hashmap_first(m->transfers))) + transfer_unref(t); + + hashmap_free(m->transfers); + + bus_verify_polkit_async_registry_free(m->polkit_registry); + + m->bus = sd_bus_flush_close_unref(m->bus); + sd_event_unref(m->event); + + return mfree(m); +} + +DEFINE_TRIVIAL_CLEANUP_FUNC(Manager*, manager_unref); + +static int manager_on_notify(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + + char buf[NOTIFY_BUFFER_MAX+1]; + struct iovec iovec = { + .iov_base = buf, + .iov_len = sizeof(buf)-1, + }; + union { + struct cmsghdr cmsghdr; + uint8_t buf[CMSG_SPACE(sizeof(struct ucred)) + + CMSG_SPACE(sizeof(int) * NOTIFY_FD_MAX)]; + } control = {}; + struct msghdr msghdr = { + .msg_iov = &iovec, + .msg_iovlen = 1, + .msg_control = &control, + .msg_controllen = sizeof(control), + }; + struct ucred *ucred = NULL; + Manager *m = userdata; + struct cmsghdr *cmsg; + char *p, *e; + Transfer *t; + Iterator i; + ssize_t n; + int r; + + n = recvmsg(fd, &msghdr, MSG_DONTWAIT|MSG_CMSG_CLOEXEC); + if (n < 0) { + if (IN_SET(errno, EAGAIN, EINTR)) + return 0; + + return -errno; + } + + cmsg_close_all(&msghdr); + + CMSG_FOREACH(cmsg, &msghdr) + if (cmsg->cmsg_level == SOL_SOCKET && + cmsg->cmsg_type == SCM_CREDENTIALS && + cmsg->cmsg_len == CMSG_LEN(sizeof(struct ucred))) + ucred = (struct ucred*) CMSG_DATA(cmsg); + + if (msghdr.msg_flags & MSG_TRUNC) { + log_warning("Got overly long notification datagram, ignoring."); + return 0; + } + + if (!ucred || ucred->pid <= 0) { + log_warning("Got notification datagram lacking credential information, ignoring."); + return 0; + } + + HASHMAP_FOREACH(t, m->transfers, i) + if (ucred->pid == t->pid) + break; + + if (!t) { + log_warning("Got notification datagram from unexpected peer, ignoring."); + return 0; + } + + buf[n] = 0; + + p = startswith(buf, "X_IMPORT_PROGRESS="); + if (!p) { + p = strstr(buf, "\nX_IMPORT_PROGRESS="); + if (!p) + return 0; + + p += 19; + } + + e = strchrnul(p, '\n'); + *e = 0; + + r = parse_percent(p); + if (r < 0) { + log_warning("Got invalid percent value, ignoring."); + return 0; + } + + t->progress_percent = (unsigned) r; + + log_debug("Got percentage from client: %u%%", t->progress_percent); + return 0; +} + +static int manager_new(Manager **ret) { + _cleanup_(manager_unrefp) Manager *m = NULL; + static const union sockaddr_union sa = { + .un.sun_family = AF_UNIX, + .un.sun_path = "/run/systemd/import/notify", + }; + int r; + + assert(ret); + + m = new0(Manager, 1); + if (!m) + return -ENOMEM; + + r = sd_event_default(&m->event); + if (r < 0) + return r; + + sd_event_set_watchdog(m->event, true); + + r = sd_bus_default_system(&m->bus); + if (r < 0) + return r; + + m->notify_fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); + if (m->notify_fd < 0) + return -errno; + + (void) mkdir_parents_label(sa.un.sun_path, 0755); + (void) sockaddr_un_unlink(&sa.un); + + if (bind(m->notify_fd, &sa.sa, SOCKADDR_UN_LEN(sa.un)) < 0) + return -errno; + + r = setsockopt_int(m->notify_fd, SOL_SOCKET, SO_PASSCRED, true); + if (r < 0) + return r; + + r = sd_event_add_io(m->event, &m->notify_event_source, m->notify_fd, EPOLLIN, manager_on_notify, m); + if (r < 0) + return r; + + *ret = TAKE_PTR(m); + + return 0; +} + +static Transfer *manager_find(Manager *m, TransferType type, const char *remote) { + Transfer *t; + Iterator i; + + assert(m); + assert(type >= 0); + assert(type < _TRANSFER_TYPE_MAX); + + HASHMAP_FOREACH(t, m->transfers, i) + if (t->type == type && streq_ptr(t->remote, remote)) + return t; + + return NULL; +} + +static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, force, read_only, r; + const char *local, *object; + Manager *m = userdata; + TransferType type; + uint32_t id; + + assert(msg); + assert(m); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.import", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); + if (r < 0) + return r; + + r = fd_verify_regular(fd); + if (r < 0) + return r; + + if (!machine_name_is_valid(local)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); + + r = setup_machine_directory(error); + if (r < 0) + return r; + + type = streq_ptr(sd_bus_message_get_member(msg), "ImportTar") ? TRANSFER_IMPORT_TAR : TRANSFER_IMPORT_RAW; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + t->force_local = force; + t->read_only = read_only; + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdin_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdin_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, force, read_only, r; + const char *local, *object; + Manager *m = userdata; + uint32_t id; + + assert(msg); + assert(m); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.import", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); + if (r < 0) + return r; + + r = fd_verify_directory(fd); + if (r < 0) + return r; + + if (!machine_name_is_valid(local)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); + + r = setup_machine_directory(error); + if (r < 0) + return r; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = TRANSFER_IMPORT_FS; + t->force_local = force; + t->read_only = read_only; + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdin_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdin_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, r; + const char *local, *object, *format; + Manager *m = userdata; + TransferType type; + uint32_t id; + + assert(msg); + assert(m); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.export", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "shs", &local, &fd, &format); + if (r < 0) + return r; + + if (!machine_name_is_valid(local)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); + + r = fd_verify_regular(fd); + if (r < 0) + return r; + + type = streq_ptr(sd_bus_message_get_member(msg), "ExportTar") ? TRANSFER_EXPORT_TAR : TRANSFER_EXPORT_RAW; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + + if (!isempty(format)) { + t->format = strdup(format); + if (!t->format) + return -ENOMEM; + } + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdout_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdout_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + const char *remote, *local, *verify, *object; + Manager *m = userdata; + ImportVerify v; + TransferType type; + int force, r; + uint32_t id; + + assert(msg); + assert(m); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.pull", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "sssb", &remote, &local, &verify, &force); + if (r < 0) + return r; + + if (!http_url_is_valid(remote)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "URL %s is invalid", remote); + + if (isempty(local)) + local = NULL; + else if (!machine_name_is_valid(local)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); + + if (isempty(verify)) + v = IMPORT_VERIFY_SIGNATURE; + else + v = import_verify_from_string(verify); + if (v < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Unknown verification mode %s", verify); + + r = setup_machine_directory(error); + if (r < 0) + return r; + + type = streq_ptr(sd_bus_message_get_member(msg), "PullTar") ? TRANSFER_PULL_TAR : TRANSFER_PULL_RAW; + + if (manager_find(m, type, remote)) + return sd_bus_error_setf(error, BUS_ERROR_TRANSFER_IN_PROGRESS, "Transfer for %s already in progress.", remote); + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + t->verify = v; + t->force_local = force; + + t->remote = strdup(remote); + if (!t->remote) + return -ENOMEM; + + if (local) { + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + } + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_list_transfers(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; + Manager *m = userdata; + Transfer *t; + Iterator i; + int r; + + assert(msg); + assert(m); + + r = sd_bus_message_new_method_return(msg, &reply); + if (r < 0) + return r; + + r = sd_bus_message_open_container(reply, 'a', "(usssdo)"); + if (r < 0) + return r; + + HASHMAP_FOREACH(t, m->transfers, i) { + + r = sd_bus_message_append( + reply, + "(usssdo)", + t->id, + transfer_type_to_string(t->type), + t->remote, + t->local, + transfer_percent_as_double(t), + t->object_path); + if (r < 0) + return r; + } + + r = sd_bus_message_close_container(reply); + if (r < 0) + return r; + + return sd_bus_send(NULL, reply, NULL); +} + +static int method_cancel(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + Transfer *t = userdata; + int r; + + assert(msg); + assert(t); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.pull", + NULL, + false, + UID_INVALID, + &t->manager->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = transfer_cancel(t); + if (r < 0) + return r; + + return sd_bus_reply_method_return(msg, NULL); +} + +static int method_cancel_transfer(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + Manager *m = userdata; + Transfer *t; + uint32_t id; + int r; + + assert(msg); + assert(m); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.pull", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "u", &id); + if (r < 0) + return r; + if (id <= 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Invalid transfer id"); + + t = hashmap_get(m->transfers, UINT32_TO_PTR(id)); + if (!t) + return sd_bus_error_setf(error, BUS_ERROR_NO_SUCH_TRANSFER, "No transfer by id %" PRIu32, id); + + r = transfer_cancel(t); + if (r < 0) + return r; + + return sd_bus_reply_method_return(msg, NULL); +} + +static int property_get_progress( + sd_bus *bus, + const char *path, + const char *interface, + const char *property, + sd_bus_message *reply, + void *userdata, + sd_bus_error *error) { + + Transfer *t = userdata; + + assert(bus); + assert(reply); + assert(t); + + return sd_bus_message_append(reply, "d", transfer_percent_as_double(t)); +} + +static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_type, transfer_type, TransferType); +static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_verify, import_verify, ImportVerify); + +static const sd_bus_vtable transfer_vtable[] = { + SD_BUS_VTABLE_START(0), + SD_BUS_PROPERTY("Id", "u", NULL, offsetof(Transfer, id), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Local", "s", NULL, offsetof(Transfer, local), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Remote", "s", NULL, offsetof(Transfer, remote), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Type", "s", property_get_type, offsetof(Transfer, type), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Verify", "s", property_get_verify, offsetof(Transfer, verify), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Progress", "d", property_get_progress, 0, 0), + SD_BUS_METHOD("Cancel", NULL, NULL, method_cancel, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_SIGNAL("LogMessage", "us", 0), + SD_BUS_VTABLE_END, +}; + +static const sd_bus_vtable manager_vtable[] = { + SD_BUS_VTABLE_START(0), + SD_BUS_METHOD("ImportTar", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ImportRaw", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ImportFileSystem", "hsbb", "uo", method_import_fs, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ExportTar", "shs", "uo", method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ExportRaw", "shs", "uo", method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("PullTar", "sssb", "uo", method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("PullRaw", "sssb", "uo", method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ListTransfers", NULL, "a(usssdo)", method_list_transfers, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("CancelTransfer", "u", NULL, method_cancel_transfer, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_SIGNAL("TransferNew", "uo", 0), + SD_BUS_SIGNAL("TransferRemoved", "uos", 0), + SD_BUS_VTABLE_END, +}; + +static int transfer_object_find(sd_bus *bus, const char *path, const char *interface, void *userdata, void **found, sd_bus_error *error) { + Manager *m = userdata; + Transfer *t; + const char *p; + uint32_t id; + int r; + + assert(bus); + assert(path); + assert(interface); + assert(found); + assert(m); + + p = startswith(path, "/org/freedesktop/import1/transfer/_"); + if (!p) + return 0; + + r = safe_atou32(p, &id); + if (r < 0 || id == 0) + return 0; + + t = hashmap_get(m->transfers, UINT32_TO_PTR(id)); + if (!t) + return 0; + + *found = t; + return 1; +} + +static int transfer_node_enumerator(sd_bus *bus, const char *path, void *userdata, char ***nodes, sd_bus_error *error) { + _cleanup_strv_free_ char **l = NULL; + Manager *m = userdata; + Transfer *t; + unsigned k = 0; + Iterator i; + + l = new0(char*, hashmap_size(m->transfers) + 1); + if (!l) + return -ENOMEM; + + HASHMAP_FOREACH(t, m->transfers, i) { + + l[k] = strdup(t->object_path); + if (!l[k]) + return -ENOMEM; + + k++; + } + + *nodes = TAKE_PTR(l); + + return 1; +} + +static int manager_add_bus_objects(Manager *m) { + int r; + + assert(m); + + r = sd_bus_add_object_vtable(m->bus, NULL, "/org/freedesktop/import1", "org.freedesktop.import1.Manager", manager_vtable, m); + if (r < 0) + return log_error_errno(r, "Failed to register object: %m"); + + r = sd_bus_add_fallback_vtable(m->bus, NULL, "/org/freedesktop/import1/transfer", "org.freedesktop.import1.Transfer", transfer_vtable, transfer_object_find, m); + if (r < 0) + return log_error_errno(r, "Failed to register object: %m"); + + r = sd_bus_add_node_enumerator(m->bus, NULL, "/org/freedesktop/import1/transfer", transfer_node_enumerator, m); + if (r < 0) + return log_error_errno(r, "Failed to add transfer enumerator: %m"); + + r = sd_bus_request_name_async(m->bus, NULL, "org.freedesktop.import1", 0, NULL, NULL); + if (r < 0) + return log_error_errno(r, "Failed to request name: %m"); + + r = sd_bus_attach_event(m->bus, m->event, 0); + if (r < 0) + return log_error_errno(r, "Failed to attach bus to event loop: %m"); + + return 0; +} + +static bool manager_check_idle(void *userdata) { + Manager *m = userdata; + + return hashmap_isempty(m->transfers); +} + +static int manager_run(Manager *m) { + assert(m); + + return bus_event_loop_with_idle( + m->event, + m->bus, + "org.freedesktop.import1", + DEFAULT_EXIT_USEC, + manager_check_idle, + m); +} + +static int run(int argc, char *argv[]) { + _cleanup_(manager_unrefp) Manager *m = NULL; + int r; + + log_setup_service(); + + umask(0022); + + if (argc != 1) { + log_error("This program takes no arguments."); + return -EINVAL; + } + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGCHLD, -1) >= 0); + + r = manager_new(&m); + if (r < 0) + return log_error_errno(r, "Failed to allocate manager object: %m"); + + r = manager_add_bus_objects(m); + if (r < 0) + return r; + + r = manager_run(m); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + return 0; +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/meson.build b/src/import/meson.build new file mode 100644 index 0000000..1c15fd8 --- /dev/null +++ b/src/import/meson.build @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: LGPL-2.1+ + +systemd_importd_sources = files(''' + importd.c +'''.split()) + +systemd_pull_sources = files(''' + pull.c + pull-raw.c + pull-raw.h + pull-tar.c + pull-tar.h + pull-job.c + pull-job.h + pull-common.c + pull-common.h + import-common.c + import-common.h + import-compress.c + import-compress.h + curl-util.c + curl-util.h + qcow2-util.c + qcow2-util.h +'''.split()) + +systemd_import_sources = files(''' + import.c + import-raw.c + import-raw.h + import-tar.c + import-tar.h + import-common.c + import-common.h + import-compress.c + import-compress.h + qcow2-util.c + qcow2-util.h +'''.split()) + +systemd_import_fs_sources = files(''' + import-fs.c + import-common.c + import-common.h +'''.split()) + +systemd_export_sources = files(''' + export.c + export-tar.c + export-tar.h + export-raw.c + export-raw.h + import-common.c + import-common.h + import-compress.c + import-compress.h +'''.split()) + +if conf.get('ENABLE_IMPORTD') == 1 + install_data('org.freedesktop.import1.conf', + install_dir : dbuspolicydir) + install_data('org.freedesktop.import1.service', + install_dir : dbussystemservicedir) + install_data('org.freedesktop.import1.policy', + install_dir : polkitpolicydir) + + install_data('import-pubring.gpg', + install_dir : rootlibexecdir) + # TODO: shouldn't this be in pkgdatadir? +endif + +tests += [ + [['src/import/test-qcow2.c', + 'src/import/qcow2-util.c', + 'src/import/qcow2-util.h'], + [libshared], + [libz], + 'HAVE_ZLIB', 'manual'], +] diff --git a/src/import/org.freedesktop.import1.conf b/src/import/org.freedesktop.import1.conf new file mode 100644 index 0000000..2fdb2ba --- /dev/null +++ b/src/import/org.freedesktop.import1.conf @@ -0,0 +1,84 @@ +<?xml version="1.0"?> <!--*-nxml-*--> +<!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN" + "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd"> + +<!-- + SPDX-License-Identifier: LGPL-2.1+ + + This file is part of systemd. + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. +--> + +<busconfig> + + <policy user="root"> + <allow own="org.freedesktop.import1"/> + <allow send_destination="org.freedesktop.import1"/> + <allow receive_sender="org.freedesktop.import1"/> + </policy> + + <policy context="default"> + <deny send_destination="org.freedesktop.import1"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Introspectable"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Peer"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Properties" + send_member="Get"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Properties" + send_member="GetAll"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ListTransfers"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="CancelTransfer"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ImportTar"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ImportRaw"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ImportFileSystem"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ExportTar"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ExportRaw"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="PullTar"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="PullRaw"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Transfer" + send_member="Cancel"/> + + <allow receive_sender="org.freedesktop.import1"/> + </policy> + +</busconfig> diff --git a/src/import/org.freedesktop.import1.policy b/src/import/org.freedesktop.import1.policy new file mode 100644 index 0000000..beea5fe --- /dev/null +++ b/src/import/org.freedesktop.import1.policy @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> <!--*-nxml-*--> +<!DOCTYPE policyconfig PUBLIC "-//freedesktop//DTD PolicyKit Policy Configuration 1.0//EN" + "http://www.freedesktop.org/standards/PolicyKit/1/policyconfig.dtd"> + +<!-- + SPDX-License-Identifier: LGPL-2.1+ + + This file is part of systemd. + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. +--> + +<policyconfig> + + <vendor>The systemd Project</vendor> + <vendor_url>http://www.freedesktop.org/wiki/Software/systemd</vendor_url> + + <action id="org.freedesktop.import1.import"> + <description gettext-domain="systemd">Import a VM or container image</description> + <message gettext-domain="systemd">Authentication is required to import a VM or container image</message> + <defaults> + <allow_any>auth_admin</allow_any> + <allow_inactive>auth_admin</allow_inactive> + <allow_active>auth_admin_keep</allow_active> + </defaults> + </action> + + <action id="org.freedesktop.import1.export"> + <description gettext-domain="systemd">Export a VM or container image</description> + <message gettext-domain="systemd">Authentication is required to export a VM or container image</message> + <defaults> + <allow_any>auth_admin</allow_any> + <allow_inactive>auth_admin</allow_inactive> + <allow_active>auth_admin_keep</allow_active> + </defaults> + </action> + + <action id="org.freedesktop.import1.pull"> + <description gettext-domain="systemd">Download a VM or container image</description> + <message gettext-domain="systemd">Authentication is required to download a VM or container image</message> + <defaults> + <allow_any>auth_admin</allow_any> + <allow_inactive>auth_admin</allow_inactive> + <allow_active>auth_admin_keep</allow_active> + </defaults> + </action> + +</policyconfig> diff --git a/src/import/org.freedesktop.import1.service b/src/import/org.freedesktop.import1.service new file mode 100644 index 0000000..34d26d0 --- /dev/null +++ b/src/import/org.freedesktop.import1.service @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: LGPL-2.1+ +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[D-BUS Service] +Name=org.freedesktop.import1 +Exec=/bin/false +User=root +SystemdService=dbus-org.freedesktop.import1.service diff --git a/src/import/pull-common.c b/src/import/pull-common.c new file mode 100644 index 0000000..acfe380 --- /dev/null +++ b/src/import/pull-common.c @@ -0,0 +1,530 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <sys/prctl.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "capability-util.h" +#include "copy.h" +#include "dirent-util.h" +#include "escape.h" +#include "fd-util.h" +#include "io-util.h" +#include "path-util.h" +#include "process-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "rlimit-util.h" +#include "rm-rf.h" +#include "signal-util.h" +#include "siphash24.h" +#include "string-util.h" +#include "strv.h" +#include "util.h" +#include "web-util.h" + +#define FILENAME_ESCAPE "/.#\"\'" +#define HASH_URL_THRESHOLD_LENGTH (_POSIX_PATH_MAX - 16) + +int pull_find_old_etags( + const char *url, + const char *image_root, + int dt, + const char *prefix, + const char *suffix, + char ***etags) { + + _cleanup_free_ char *escaped_url = NULL; + _cleanup_closedir_ DIR *d = NULL; + _cleanup_strv_free_ char **l = NULL; + struct dirent *de; + int r; + + assert(url); + assert(etags); + + if (!image_root) + image_root = "/var/lib/machines"; + + escaped_url = xescape(url, FILENAME_ESCAPE); + if (!escaped_url) + return -ENOMEM; + + d = opendir(image_root); + if (!d) { + if (errno == ENOENT) { + *etags = NULL; + return 0; + } + + return -errno; + } + + FOREACH_DIRENT_ALL(de, d, return -errno) { + const char *a, *b; + char *u; + + if (de->d_type != DT_UNKNOWN && + de->d_type != dt) + continue; + + if (prefix) { + a = startswith(de->d_name, prefix); + if (!a) + continue; + } else + a = de->d_name; + + a = startswith(a, escaped_url); + if (!a) + continue; + + a = startswith(a, "."); + if (!a) + continue; + + if (suffix) { + b = endswith(de->d_name, suffix); + if (!b) + continue; + } else + b = strchr(de->d_name, 0); + + if (a >= b) + continue; + + r = cunescape_length(a, b - a, 0, &u); + if (r < 0) + return r; + + if (!http_etag_is_valid(u)) { + free(u); + continue; + } + + r = strv_consume(&l, u); + if (r < 0) + return r; + } + + *etags = TAKE_PTR(l); + + return 0; +} + +int pull_make_local_copy(const char *final, const char *image_root, const char *local, bool force_local) { + const char *p; + int r; + + assert(final); + assert(local); + + if (!image_root) + image_root = "/var/lib/machines"; + + p = strjoina(image_root, "/", local); + + if (force_local) + (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + r = btrfs_subvol_snapshot(final, p, + BTRFS_SNAPSHOT_QUOTA| + BTRFS_SNAPSHOT_FALLBACK_COPY| + BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| + BTRFS_SNAPSHOT_RECURSIVE); + if (r < 0) + return log_error_errno(r, "Failed to create local image: %m"); + + log_info("Created new local image '%s'.", local); + + return 0; +} + +static int hash_url(const char *url, char **ret) { + uint64_t h; + static const sd_id128_t k = SD_ID128_ARRAY(df,89,16,87,01,cc,42,30,98,ab,4a,19,a6,a5,63,4f); + + assert(url); + + h = siphash24(url, strlen(url), k.bytes); + if (asprintf(ret, "%"PRIx64, h) < 0) + return -ENOMEM; + + return 0; +} + +int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret) { + _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL; + char *path; + + assert(url); + assert(ret); + + if (!image_root) + image_root = "/var/lib/machines"; + + escaped_url = xescape(url, FILENAME_ESCAPE); + if (!escaped_url) + return -ENOMEM; + + if (etag) { + escaped_etag = xescape(etag, FILENAME_ESCAPE); + if (!escaped_etag) + return -ENOMEM; + } + + path = strjoin(image_root, "/", strempty(prefix), escaped_url, escaped_etag ? "." : "", + strempty(escaped_etag), strempty(suffix)); + if (!path) + return -ENOMEM; + + /* URLs might make the path longer than the maximum allowed length for a file name. + * When that happens, a URL hash is used instead. Paths returned by this function + * can be later used with tempfn_random() which adds 16 bytes to the resulting name. */ + if (strlen(path) >= HASH_URL_THRESHOLD_LENGTH) { + _cleanup_free_ char *hash = NULL; + int r; + + free(path); + + r = hash_url(url, &hash); + if (r < 0) + return r; + + path = strjoin(image_root, "/", strempty(prefix), hash, escaped_etag ? "." : "", + strempty(escaped_etag), strempty(suffix)); + if (!path) + return -ENOMEM; + } + + *ret = path; + return 0; +} + +int pull_make_auxiliary_job( + PullJob **ret, + const char *url, + int (*strip_suffixes)(const char *name, char **ret), + const char *suffix, + CurlGlue *glue, + PullJobFinished on_finished, + void *userdata) { + + _cleanup_free_ char *last_component = NULL, *ll = NULL, *auxiliary_url = NULL; + _cleanup_(pull_job_unrefp) PullJob *job = NULL; + const char *q; + int r; + + assert(ret); + assert(url); + assert(strip_suffixes); + assert(glue); + + r = import_url_last_component(url, &last_component); + if (r < 0) + return r; + + r = strip_suffixes(last_component, &ll); + if (r < 0) + return r; + + q = strjoina(ll, suffix); + + r = import_url_change_last_component(url, q, &auxiliary_url); + if (r < 0) + return r; + + r = pull_job_new(&job, auxiliary_url, glue, userdata); + if (r < 0) + return r; + + job->on_finished = on_finished; + job->compressed_max = job->uncompressed_max = 1ULL * 1024ULL * 1024ULL; + + *ret = TAKE_PTR(job); + + return 0; +} + +int pull_make_verification_jobs( + PullJob **ret_checksum_job, + PullJob **ret_signature_job, + ImportVerify verify, + const char *url, + CurlGlue *glue, + PullJobFinished on_finished, + void *userdata) { + + _cleanup_(pull_job_unrefp) PullJob *checksum_job = NULL, *signature_job = NULL; + int r; + const char *chksums = NULL; + + assert(ret_checksum_job); + assert(ret_signature_job); + assert(verify >= 0); + assert(verify < _IMPORT_VERIFY_MAX); + assert(url); + assert(glue); + + if (verify != IMPORT_VERIFY_NO) { + _cleanup_free_ char *checksum_url = NULL, *fn = NULL; + + /* Queue jobs for the checksum file for the image. */ + r = import_url_last_component(url, &fn); + if (r < 0) + return r; + + chksums = strjoina(fn, ".sha256"); + + r = import_url_change_last_component(url, chksums, &checksum_url); + if (r < 0) + return r; + + r = pull_job_new(&checksum_job, checksum_url, glue, userdata); + if (r < 0) + return r; + + checksum_job->on_finished = on_finished; + checksum_job->uncompressed_max = checksum_job->compressed_max = 1ULL * 1024ULL * 1024ULL; + } + + if (verify == IMPORT_VERIFY_SIGNATURE) { + _cleanup_free_ char *signature_url = NULL; + + /* Queue job for the SHA256SUMS.gpg file for the image. */ + r = import_url_change_last_component(url, "SHA256SUMS.gpg", &signature_url); + if (r < 0) + return r; + + r = pull_job_new(&signature_job, signature_url, glue, userdata); + if (r < 0) + return r; + + signature_job->on_finished = on_finished; + signature_job->uncompressed_max = signature_job->compressed_max = 1ULL * 1024ULL * 1024ULL; + } + + *ret_checksum_job = checksum_job; + *ret_signature_job = signature_job; + + checksum_job = signature_job = NULL; + + return 0; +} + +static int verify_one(PullJob *checksum_job, PullJob *job) { + _cleanup_free_ char *fn = NULL; + const char *line, *p; + int r; + + assert(checksum_job); + + if (!job) + return 0; + + assert(IN_SET(job->state, PULL_JOB_DONE, PULL_JOB_FAILED)); + + /* Don't verify the checksum if we didn't actually successfully download something new */ + if (job->state != PULL_JOB_DONE) + return 0; + if (job->error != 0) + return 0; + if (job->etag_exists) + return 0; + + assert(job->calc_checksum); + assert(job->checksum); + + r = import_url_last_component(job->url, &fn); + if (r < 0) + return log_oom(); + + if (!filename_is_valid(fn)) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "Cannot verify checksum, could not determine server-side file name."); + + line = strjoina(job->checksum, " *", fn, "\n"); + + p = memmem(checksum_job->payload, + checksum_job->payload_size, + line, + strlen(line)); + + if (!p) { + line = strjoina(job->checksum, " ", fn, "\n"); + + p = memmem(checksum_job->payload, + checksum_job->payload_size, + line, + strlen(line)); + } + + if (!p || (p != (char*) checksum_job->payload && p[-1] != '\n')) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "DOWNLOAD INVALID: Checksum of %s file did not checkout, file has been tampered with.", fn); + + log_info("SHA256 checksum of %s is valid.", job->url); + return 1; +} + +int pull_verify(PullJob *main_job, + PullJob *roothash_job, + PullJob *settings_job, + PullJob *checksum_job, + PullJob *signature_job) { + + _cleanup_close_pair_ int gpg_pipe[2] = { -1, -1 }; + _cleanup_close_ int sig_file = -1; + char sig_file_path[] = "/tmp/sigXXXXXX", gpg_home[] = "/tmp/gpghomeXXXXXX"; + _cleanup_(sigkill_waitp) pid_t pid = 0; + bool gpg_home_created = false; + int r; + + assert(main_job); + assert(main_job->state == PULL_JOB_DONE); + + if (!checksum_job) + return 0; + + assert(main_job->calc_checksum); + assert(main_job->checksum); + + assert(checksum_job->state == PULL_JOB_DONE); + + if (!checksum_job->payload || checksum_job->payload_size <= 0) { + log_error("Checksum is empty, cannot verify."); + return -EBADMSG; + } + + r = verify_one(checksum_job, main_job); + if (r < 0) + return r; + + r = verify_one(checksum_job, roothash_job); + if (r < 0) + return r; + + r = verify_one(checksum_job, settings_job); + if (r < 0) + return r; + + if (!signature_job) + return 0; + + if (checksum_job->style == VERIFICATION_PER_FILE) + signature_job = checksum_job; + + assert(signature_job->state == PULL_JOB_DONE); + + if (!signature_job->payload || signature_job->payload_size <= 0) { + log_error("Signature is empty, cannot verify."); + return -EBADMSG; + } + + r = pipe2(gpg_pipe, O_CLOEXEC); + if (r < 0) + return log_error_errno(errno, "Failed to create pipe for gpg: %m"); + + sig_file = mkostemp(sig_file_path, O_RDWR); + if (sig_file < 0) + return log_error_errno(errno, "Failed to create temporary file: %m"); + + r = loop_write(sig_file, signature_job->payload, signature_job->payload_size, false); + if (r < 0) { + log_error_errno(r, "Failed to write to temporary file: %m"); + goto finish; + } + + if (!mkdtemp(gpg_home)) { + r = log_error_errno(errno, "Failed to create tempory home for gpg: %m"); + goto finish; + } + + gpg_home_created = true; + + r = safe_fork("(gpg)", FORK_RESET_SIGNALS|FORK_DEATHSIG|FORK_LOG, &pid); + if (r < 0) + return r; + if (r == 0) { + const char *cmd[] = { + "gpg", + "--no-options", + "--no-default-keyring", + "--no-auto-key-locate", + "--no-auto-check-trustdb", + "--batch", + "--trust-model=always", + NULL, /* --homedir= */ + NULL, /* --keyring= */ + NULL, /* --verify */ + NULL, /* signature file */ + NULL, /* dash */ + NULL /* trailing NULL */ + }; + unsigned k = ELEMENTSOF(cmd) - 6; + + /* Child */ + + gpg_pipe[1] = safe_close(gpg_pipe[1]); + + r = rearrange_stdio(gpg_pipe[0], -1, STDERR_FILENO); + if (r < 0) { + log_error_errno(r, "Failed to rearrange stdin/stdout: %m"); + _exit(EXIT_FAILURE); + } + + (void) rlimit_nofile_safe(); + + cmd[k++] = strjoina("--homedir=", gpg_home); + + /* We add the user keyring only to the command line + * arguments, if it's around since gpg fails + * otherwise. */ + if (access(USER_KEYRING_PATH, F_OK) >= 0) + cmd[k++] = "--keyring=" USER_KEYRING_PATH; + else + cmd[k++] = "--keyring=" VENDOR_KEYRING_PATH; + + cmd[k++] = "--verify"; + if (checksum_job->style == VERIFICATION_PER_DIRECTORY) { + cmd[k++] = sig_file_path; + cmd[k++] = "-"; + cmd[k++] = NULL; + } + + execvp("gpg2", (char * const *) cmd); + execvp("gpg", (char * const *) cmd); + log_error_errno(errno, "Failed to execute gpg: %m"); + _exit(EXIT_FAILURE); + } + + gpg_pipe[0] = safe_close(gpg_pipe[0]); + + r = loop_write(gpg_pipe[1], checksum_job->payload, checksum_job->payload_size, false); + if (r < 0) { + log_error_errno(r, "Failed to write to pipe: %m"); + goto finish; + } + + gpg_pipe[1] = safe_close(gpg_pipe[1]); + + r = wait_for_terminate_and_check("gpg", pid, WAIT_LOG_ABNORMAL); + pid = 0; + if (r < 0) + goto finish; + if (r != EXIT_SUCCESS) { + log_error("DOWNLOAD INVALID: Signature verification failed."); + r = -EBADMSG; + } else { + log_info("Signature verification succeeded."); + r = 0; + } + +finish: + (void) unlink(sig_file_path); + + if (gpg_home_created) + (void) rm_rf(gpg_home, REMOVE_ROOT|REMOVE_PHYSICAL); + + return r; +} diff --git a/src/import/pull-common.h b/src/import/pull-common.h new file mode 100644 index 0000000..65f239d --- /dev/null +++ b/src/import/pull-common.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include <stdbool.h> + +#include "import-util.h" +#include "pull-job.h" + +int pull_make_local_copy(const char *final, const char *root, const char *local, bool force_local); + +int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags); + +int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret); + +int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, CurlGlue *glue, PullJobFinished on_finished, void *userdata); +int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); + +int pull_verify(PullJob *main_job, PullJob *roothash_job, PullJob *settings_job, PullJob *checksum_job, PullJob *signature_job); diff --git a/src/import/pull-job.c b/src/import/pull-job.c new file mode 100644 index 0000000..6881bd6 --- /dev/null +++ b/src/import/pull-job.c @@ -0,0 +1,636 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <sys/xattr.h> + +#include "alloc-util.h" +#include "fd-util.h" +#include "gcrypt-util.h" +#include "hexdecoct.h" +#include "import-util.h" +#include "io-util.h" +#include "machine-pool.h" +#include "parse-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "string-util.h" +#include "strv.h" +#include "xattr-util.h" + +PullJob* pull_job_unref(PullJob *j) { + if (!j) + return NULL; + + curl_glue_remove_and_free(j->glue, j->curl); + curl_slist_free_all(j->request_header); + + safe_close(j->disk_fd); + + import_compress_free(&j->compress); + + if (j->checksum_context) + gcry_md_close(j->checksum_context); + + free(j->url); + free(j->etag); + strv_free(j->old_etags); + free(j->payload); + free(j->checksum); + + return mfree(j); +} + +static void pull_job_finish(PullJob *j, int ret) { + assert(j); + + if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) + return; + + if (ret == 0) { + j->state = PULL_JOB_DONE; + j->progress_percent = 100; + log_info("Download of %s complete.", j->url); + } else { + j->state = PULL_JOB_FAILED; + j->error = ret; + } + + if (j->on_finished) + j->on_finished(j); +} + +static int pull_job_restart(PullJob *j) { + int r; + char *chksum_url = NULL; + + r = import_url_change_last_component(j->url, "SHA256SUMS", &chksum_url); + if (r < 0) + return r; + + free(j->url); + j->url = chksum_url; + j->state = PULL_JOB_INIT; + j->payload = mfree(j->payload); + j->payload_size = 0; + j->payload_allocated = 0; + j->written_compressed = 0; + j->written_uncompressed = 0; + + r = pull_job_begin(j); + if (r < 0) + return r; + + return 0; +} + +void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { + PullJob *j = NULL; + CURLcode code; + long status; + int r; + + if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK) + return; + + if (!j || IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) + return; + + if (result != CURLE_OK) { + log_error("Transfer failed: %s", curl_easy_strerror(result)); + r = -EIO; + goto finish; + } + + code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); + if (code != CURLE_OK) { + log_error("Failed to retrieve response code: %s", curl_easy_strerror(code)); + r = -EIO; + goto finish; + } else if (status == 304) { + log_info("Image already downloaded. Skipping download."); + j->etag_exists = true; + r = 0; + goto finish; + } else if (status >= 300) { + if (status == 404 && j->style == VERIFICATION_PER_FILE) { + + /* retry pull job with SHA256SUMS file */ + r = pull_job_restart(j); + if (r < 0) + goto finish; + + code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status); + if (code != CURLE_OK) { + log_error("Failed to retrieve response code: %s", curl_easy_strerror(code)); + r = -EIO; + goto finish; + } + + if (status == 0) { + j->style = VERIFICATION_PER_DIRECTORY; + return; + } + } + + log_error("HTTP request to %s failed with code %li.", j->url, status); + r = -EIO; + goto finish; + } else if (status < 200) { + log_error("HTTP request to %s finished with unexpected code %li.", j->url, status); + r = -EIO; + goto finish; + } + + if (j->state != PULL_JOB_RUNNING) { + log_error("Premature connection termination."); + r = -EIO; + goto finish; + } + + if (j->content_length != (uint64_t) -1 && + j->content_length != j->written_compressed) { + log_error("Download truncated."); + r = -EIO; + goto finish; + } + + if (j->checksum_context) { + uint8_t *k; + + k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256); + if (!k) { + log_error("Failed to get checksum."); + r = -EIO; + goto finish; + } + + j->checksum = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256)); + if (!j->checksum) { + r = log_oom(); + goto finish; + } + + log_debug("SHA256 of %s is %s.", j->url, j->checksum); + } + + if (j->disk_fd >= 0 && j->allow_sparse) { + /* Make sure the file size is right, in case the file was + * sparse and we just seeked for the last part */ + + if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) { + r = log_error_errno(errno, "Failed to truncate file: %m"); + goto finish; + } + + if (j->etag) + (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0); + if (j->url) + (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0); + + if (j->mtime != 0) { + struct timespec ut[2]; + + timespec_store(&ut[0], j->mtime); + ut[1] = ut[0]; + (void) futimens(j->disk_fd, ut); + + (void) fd_setcrtime(j->disk_fd, j->mtime); + } + } + + r = 0; + +finish: + pull_job_finish(j, r); +} + +static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) { + PullJob *j = userdata; + ssize_t n; + + assert(j); + assert(p); + + if (sz <= 0) + return 0; + + if (j->written_uncompressed + sz < j->written_uncompressed) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), + "File too large, overflow"); + + if (j->written_uncompressed + sz > j->uncompressed_max) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), + "File overly large, refusing"); + + if (j->disk_fd >= 0) { + + if (j->allow_sparse) + n = sparse_write(j->disk_fd, p, sz, 64); + else { + n = write(j->disk_fd, p, sz); + if (n < 0) + n = -errno; + } + if (n < 0) + return log_error_errno((int) n, "Failed to write file: %m"); + if ((size_t) n < sz) + return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); + } else { + + if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) + return log_oom(); + + memcpy(j->payload + j->payload_size, p, sz); + j->payload_size += sz; + } + + j->written_uncompressed += sz; + + return 0; +} + +static int pull_job_write_compressed(PullJob *j, void *p, size_t sz) { + int r; + + assert(j); + assert(p); + + if (sz <= 0) + return 0; + + if (j->written_compressed + sz < j->written_compressed) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); + + if (j->written_compressed + sz > j->compressed_max) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); + + if (j->content_length != (uint64_t) -1 && + j->written_compressed + sz > j->content_length) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), + "Content length incorrect."); + + if (j->checksum_context) + gcry_md_write(j->checksum_context, p, sz); + + r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j); + if (r < 0) + return r; + + j->written_compressed += sz; + + return 0; +} + +static int pull_job_open_disk(PullJob *j) { + int r; + + assert(j); + + if (j->on_open_disk) { + r = j->on_open_disk(j); + if (r < 0) + return r; + } + + if (j->disk_fd >= 0) { + /* Check if we can do sparse files */ + + if (lseek(j->disk_fd, SEEK_SET, 0) == 0) + j->allow_sparse = true; + else { + if (errno != ESPIPE) + return log_error_errno(errno, "Failed to seek on file descriptor: %m"); + + j->allow_sparse = false; + } + } + + if (j->calc_checksum) { + initialize_libgcrypt(false); + + if (gcry_md_open(&j->checksum_context, GCRY_MD_SHA256, 0) != 0) + return log_error_errno(SYNTHETIC_ERRNO(EIO), + "Failed to initialize hash context."); + } + + return 0; +} + +static int pull_job_detect_compression(PullJob *j) { + _cleanup_free_ uint8_t *stub = NULL; + size_t stub_size; + + int r; + + assert(j); + + r = import_uncompress_detect(&j->compress, j->payload, j->payload_size); + if (r < 0) + return log_error_errno(r, "Failed to initialize compressor: %m"); + if (r == 0) + return 0; + + log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type)); + + r = pull_job_open_disk(j); + if (r < 0) + return r; + + /* Now, take the payload we read so far, and decompress it */ + stub = j->payload; + stub_size = j->payload_size; + + j->payload = NULL; + j->payload_size = 0; + j->payload_allocated = 0; + + j->state = PULL_JOB_RUNNING; + + r = pull_job_write_compressed(j, stub, stub_size); + if (r < 0) + return r; + + return 0; +} + +static size_t pull_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) { + PullJob *j = userdata; + size_t sz = size * nmemb; + int r; + + assert(contents); + assert(j); + + switch (j->state) { + + case PULL_JOB_ANALYZING: + /* Let's first check what it actually is */ + + if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) { + r = log_oom(); + goto fail; + } + + memcpy(j->payload + j->payload_size, contents, sz); + j->payload_size += sz; + + r = pull_job_detect_compression(j); + if (r < 0) + goto fail; + + break; + + case PULL_JOB_RUNNING: + + r = pull_job_write_compressed(j, contents, sz); + if (r < 0) + goto fail; + + break; + + case PULL_JOB_DONE: + case PULL_JOB_FAILED: + r = -ESTALE; + goto fail; + + default: + assert_not_reached("Impossible state."); + } + + return sz; + +fail: + pull_job_finish(j, r); + return 0; +} + +static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) { + PullJob *j = userdata; + size_t sz = size * nmemb; + _cleanup_free_ char *length = NULL, *last_modified = NULL; + char *etag; + int r; + + assert(contents); + assert(j); + + if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) { + r = -ESTALE; + goto fail; + } + + assert(j->state == PULL_JOB_ANALYZING); + + r = curl_header_strdup(contents, sz, "ETag:", &etag); + if (r < 0) { + log_oom(); + goto fail; + } + if (r > 0) { + free(j->etag); + j->etag = etag; + + if (strv_contains(j->old_etags, j->etag)) { + log_info("Image already downloaded. Skipping download."); + j->etag_exists = true; + pull_job_finish(j, 0); + return sz; + } + + return sz; + } + + r = curl_header_strdup(contents, sz, "Content-Length:", &length); + if (r < 0) { + log_oom(); + goto fail; + } + if (r > 0) { + (void) safe_atou64(length, &j->content_length); + + if (j->content_length != (uint64_t) -1) { + char bytes[FORMAT_BYTES_MAX]; + + if (j->content_length > j->compressed_max) { + log_error("Content too large."); + r = -EFBIG; + goto fail; + } + + log_info("Downloading %s for %s.", format_bytes(bytes, sizeof(bytes), j->content_length), j->url); + } + + return sz; + } + + r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified); + if (r < 0) { + log_oom(); + goto fail; + } + if (r > 0) { + (void) curl_parse_http_time(last_modified, &j->mtime); + return sz; + } + + if (j->on_header) { + r = j->on_header(j, contents, sz); + if (r < 0) + goto fail; + } + + return sz; + +fail: + pull_job_finish(j, r); + return 0; +} + +static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { + PullJob *j = userdata; + unsigned percent; + usec_t n; + + assert(j); + + if (dltotal <= 0) + return 0; + + percent = ((100 * dlnow) / dltotal); + n = now(CLOCK_MONOTONIC); + + if (n > j->last_status_usec + USEC_PER_SEC && + percent != j->progress_percent && + dlnow < dltotal) { + char buf[FORMAT_TIMESPAN_MAX]; + + if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) { + char y[FORMAT_BYTES_MAX]; + usec_t left, done; + + done = n - j->start_usec; + left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done; + + log_info("Got %u%% of %s. %s left at %s/s.", + percent, + j->url, + format_timespan(buf, sizeof(buf), left, USEC_PER_SEC), + format_bytes(y, sizeof(y), (uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC)))); + } else + log_info("Got %u%% of %s.", percent, j->url); + + j->progress_percent = percent; + j->last_status_usec = n; + + if (j->on_progress) + j->on_progress(j); + } + + return 0; +} + +int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) { + _cleanup_(pull_job_unrefp) PullJob *j = NULL; + _cleanup_free_ char *u = NULL; + + assert(url); + assert(glue); + assert(ret); + + u = strdup(url); + if (!u) + return -ENOMEM; + + j = new(PullJob, 1); + if (!j) + return -ENOMEM; + + *j = (PullJob) { + .state = PULL_JOB_INIT, + .disk_fd = -1, + .userdata = userdata, + .glue = glue, + .content_length = (uint64_t) -1, + .start_usec = now(CLOCK_MONOTONIC), + .compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */ + .uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */ + .style = VERIFICATION_STYLE_UNSET, + .url = TAKE_PTR(u), + }; + + *ret = TAKE_PTR(j); + + return 0; +} + +int pull_job_begin(PullJob *j) { + int r; + + assert(j); + + if (j->state != PULL_JOB_INIT) + return -EBUSY; + + r = curl_glue_make(&j->curl, j->url, j); + if (r < 0) + return r; + + if (!strv_isempty(j->old_etags)) { + _cleanup_free_ char *cc = NULL, *hdr = NULL; + + cc = strv_join(j->old_etags, ", "); + if (!cc) + return -ENOMEM; + + hdr = strappend("If-None-Match: ", cc); + if (!hdr) + return -ENOMEM; + + if (!j->request_header) { + j->request_header = curl_slist_new(hdr, NULL); + if (!j->request_header) + return -ENOMEM; + } else { + struct curl_slist *l; + + l = curl_slist_append(j->request_header, hdr); + if (!l) + return -ENOMEM; + + j->request_header = l; + } + } + + if (j->request_header) { + if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK) + return -EIO; + } + + if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK) + return -EIO; + + r = curl_glue_add(j->glue, j->curl); + if (r < 0) + return r; + + j->state = PULL_JOB_ANALYZING; + + return 0; +} diff --git a/src/import/pull-job.h b/src/import/pull-job.h new file mode 100644 index 0000000..c907e74 --- /dev/null +++ b/src/import/pull-job.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include <gcrypt.h> + +#include "curl-util.h" +#include "import-compress.h" +#include "macro.h" + +typedef struct PullJob PullJob; + +typedef void (*PullJobFinished)(PullJob *job); +typedef int (*PullJobOpenDisk)(PullJob *job); +typedef int (*PullJobHeader)(PullJob *job, const char *header, size_t sz); +typedef void (*PullJobProgress)(PullJob *job); + +typedef enum PullJobState { + PULL_JOB_INIT, + PULL_JOB_ANALYZING, /* Still reading into ->payload, to figure out what we have */ + PULL_JOB_RUNNING, /* Writing to destination */ + PULL_JOB_DONE, + PULL_JOB_FAILED, + _PULL_JOB_STATE_MAX, + _PULL_JOB_STATE_INVALID = -1, +} PullJobState; + +typedef enum VerificationStyle { + VERIFICATION_STYLE_UNSET, + VERIFICATION_PER_FILE, /* SuSE-style ".sha256" files with inline signature */ + VERIFICATION_PER_DIRECTORY, /* Ubuntu-style SHA256SUM files with detach SHA256SUM.gpg signatures */ +} VerificationStyle; + +#define PULL_JOB_IS_COMPLETE(j) (IN_SET((j)->state, PULL_JOB_DONE, PULL_JOB_FAILED)) + +struct PullJob { + PullJobState state; + int error; + + char *url; + + void *userdata; + PullJobFinished on_finished; + PullJobOpenDisk on_open_disk; + PullJobHeader on_header; + PullJobProgress on_progress; + + CurlGlue *glue; + CURL *curl; + struct curl_slist *request_header; + + char *etag; + char **old_etags; + bool etag_exists; + + uint64_t content_length; + uint64_t written_compressed; + uint64_t written_uncompressed; + + uint64_t uncompressed_max; + uint64_t compressed_max; + + uint8_t *payload; + size_t payload_size; + size_t payload_allocated; + + int disk_fd; + + usec_t mtime; + + ImportCompress compress; + + unsigned progress_percent; + usec_t start_usec; + usec_t last_status_usec; + + bool allow_sparse; + + bool calc_checksum; + gcry_md_hd_t checksum_context; + + char *checksum; + + VerificationStyle style; +}; + +int pull_job_new(PullJob **job, const char *url, CurlGlue *glue, void *userdata); +PullJob* pull_job_unref(PullJob *job); + +int pull_job_begin(PullJob *j); + +void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result); + +DEFINE_TRIVIAL_CLEANUP_FUNC(PullJob*, pull_job_unref); diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c new file mode 100644 index 0000000..3a3e015 --- /dev/null +++ b/src/import/pull-raw.c @@ -0,0 +1,750 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <curl/curl.h> +#include <linux/fs.h> +#include <sys/xattr.h> + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "chattr-util.h" +#include "copy.h" +#include "curl-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-util.h" +#include "macro.h" +#include "mkdir.h" +#include "path-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "pull-raw.h" +#include "qcow2-util.h" +#include "rm-rf.h" +#include "string-util.h" +#include "strv.h" +#include "tmpfile-util.h" +#include "utf8.h" +#include "util.h" +#include "web-util.h" + +typedef enum RawProgress { + RAW_DOWNLOADING, + RAW_VERIFYING, + RAW_UNPACKING, + RAW_FINALIZING, + RAW_COPYING, +} RawProgress; + +struct RawPull { + sd_event *event; + CurlGlue *glue; + + char *image_root; + + PullJob *raw_job; + PullJob *roothash_job; + PullJob *settings_job; + PullJob *checksum_job; + PullJob *signature_job; + + RawPullFinished on_finished; + void *userdata; + + char *local; + bool force_local; + bool settings; + bool roothash; + + char *final_path; + char *temp_path; + + char *settings_path; + char *settings_temp_path; + + char *roothash_path; + char *roothash_temp_path; + + ImportVerify verify; +}; + +RawPull* raw_pull_unref(RawPull *i) { + if (!i) + return NULL; + + pull_job_unref(i->raw_job); + pull_job_unref(i->settings_job); + pull_job_unref(i->roothash_job); + pull_job_unref(i->checksum_job); + pull_job_unref(i->signature_job); + + curl_glue_unref(i->glue); + sd_event_unref(i->event); + + if (i->temp_path) { + (void) unlink(i->temp_path); + free(i->temp_path); + } + + if (i->roothash_temp_path) { + (void) unlink(i->roothash_temp_path); + free(i->roothash_temp_path); + } + + if (i->settings_temp_path) { + (void) unlink(i->settings_temp_path); + free(i->settings_temp_path); + } + + free(i->final_path); + free(i->roothash_path); + free(i->settings_path); + free(i->image_root); + free(i->local); + return mfree(i); +} + +int raw_pull_new( + RawPull **ret, + sd_event *event, + const char *image_root, + RawPullFinished on_finished, + void *userdata) { + + _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL; + _cleanup_(sd_event_unrefp) sd_event *e = NULL; + _cleanup_(raw_pull_unrefp) RawPull *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + if (event) + e = sd_event_ref(event); + else { + r = sd_event_default(&e); + if (r < 0) + return r; + } + + r = curl_glue_new(&g, e); + if (r < 0) + return r; + + i = new(RawPull, 1); + if (!i) + return -ENOMEM; + + *i = (RawPull) { + .on_finished = on_finished, + .userdata = userdata, + .image_root = TAKE_PTR(root), + .event = TAKE_PTR(e), + .glue = TAKE_PTR(g), + }; + + i->glue->on_finished = pull_job_curl_on_finished; + i->glue->userdata = i; + + *ret = TAKE_PTR(i); + + return 0; +} + +static void raw_pull_report_progress(RawPull *i, RawProgress p) { + unsigned percent; + + assert(i); + + switch (p) { + + case RAW_DOWNLOADING: { + unsigned remain = 80; + + percent = 0; + + if (i->settings_job) { + percent += i->settings_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->roothash_job) { + percent += i->roothash_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->checksum_job) { + percent += i->checksum_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->signature_job) { + percent += i->signature_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->raw_job) + percent += i->raw_job->progress_percent * remain / 100; + break; + } + + case RAW_VERIFYING: + percent = 80; + break; + + case RAW_UNPACKING: + percent = 85; + break; + + case RAW_FINALIZING: + percent = 90; + break; + + case RAW_COPYING: + percent = 95; + break; + + default: + assert_not_reached("Unknown progress state"); + } + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_debug("Combined progress %u%%", percent); +} + +static int raw_pull_maybe_convert_qcow2(RawPull *i) { + _cleanup_close_ int converted_fd = -1; + _cleanup_free_ char *t = NULL; + int r; + + assert(i); + assert(i->raw_job); + + r = qcow2_detect(i->raw_job->disk_fd); + if (r < 0) + return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); + if (r == 0) + return 0; + + /* This is a QCOW2 image, let's convert it */ + r = tempfn_random(i->final_path, NULL, &t); + if (r < 0) + return log_oom(); + + converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (converted_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", t); + + r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL, NULL); + if (r < 0) + log_warning_errno(r, "Failed to set file attributes on %s: %m", t); + + log_info("Unpacking QCOW2 file."); + + r = qcow2_convert(i->raw_job->disk_fd, converted_fd); + if (r < 0) { + unlink(t); + return log_error_errno(r, "Failed to convert qcow2 image: %m"); + } + + (void) unlink(i->temp_path); + free_and_replace(i->temp_path, t); + + safe_close(i->raw_job->disk_fd); + i->raw_job->disk_fd = TAKE_FD(converted_fd); + + return 1; +} + +static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) { + int r; + + assert(i); + assert(field); + + if (*field) + return 0; + + assert(i->raw_job); + + r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", suffix, field); + if (r < 0) + return log_oom(); + + return 1; +} + +static int raw_pull_copy_auxiliary_file( + RawPull *i, + const char *suffix, + char **path) { + + const char *local; + int r; + + assert(i); + assert(suffix); + assert(path); + + r = raw_pull_determine_path(i, suffix, path); + if (r < 0) + return r; + + local = strjoina(i->image_root, "/", i->local, suffix); + + r = copy_file_atomic(*path, local, 0644, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0)); + if (r == -EEXIST) + log_warning_errno(r, "File %s already exists, not replacing.", local); + else if (r == -ENOENT) + log_debug_errno(r, "Skipping creation of auxiliary file, since none was found."); + else if (r < 0) + log_warning_errno(r, "Failed to copy file %s, ignoring: %m", local); + else + log_info("Created new file %s.", local); + + return 0; +} + +static int raw_pull_make_local_copy(RawPull *i) { + _cleanup_free_ char *tp = NULL; + _cleanup_close_ int dfd = -1; + const char *p; + int r; + + assert(i); + assert(i->raw_job); + + if (!i->local) + return 0; + + if (i->raw_job->etag_exists) { + /* We have downloaded this one previously, reopen it */ + + assert(i->raw_job->disk_fd < 0); + + i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC); + if (i->raw_job->disk_fd < 0) + return log_error_errno(errno, "Failed to open vendor image: %m"); + } else { + /* We freshly downloaded the image, use it */ + + assert(i->raw_job->disk_fd >= 0); + + if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1) + return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m"); + } + + p = strjoina(i->image_root, "/", i->local, ".raw"); + + if (i->force_local) + (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + r = tempfn_random(p, NULL, &tp); + if (r < 0) + return log_oom(); + + dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (dfd < 0) + return log_error_errno(errno, "Failed to create writable copy of image: %m"); + + /* Turn off COW writing. This should greatly improve + * performance on COW file systems like btrfs, since it + * reduces fragmentation caused by not allowing in-place + * writes. */ + r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL, NULL); + if (r < 0) + log_warning_errno(r, "Failed to set file attributes on %s: %m", tp); + + r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, COPY_REFLINK); + if (r < 0) { + unlink(tp); + return log_error_errno(r, "Failed to make writable copy of image: %m"); + } + + (void) copy_times(i->raw_job->disk_fd, dfd); + (void) copy_xattr(i->raw_job->disk_fd, dfd); + + dfd = safe_close(dfd); + + r = rename(tp, p); + if (r < 0) { + r = log_error_errno(errno, "Failed to move writable image into place: %m"); + unlink(tp); + return r; + } + + log_info("Created new local image '%s'.", i->local); + + if (i->roothash) { + r = raw_pull_copy_auxiliary_file(i, ".roothash", &i->roothash_path); + if (r < 0) + return r; + } + + if (i->settings) { + r = raw_pull_copy_auxiliary_file(i, ".nspawn", &i->settings_path); + if (r < 0) + return r; + } + + return 0; +} + +static bool raw_pull_is_done(RawPull *i) { + assert(i); + assert(i->raw_job); + + if (!PULL_JOB_IS_COMPLETE(i->raw_job)) + return false; + if (i->roothash_job && !PULL_JOB_IS_COMPLETE(i->roothash_job)) + return false; + if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job)) + return false; + if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job)) + return false; + if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job)) + return false; + + return true; +} + +static int raw_pull_rename_auxiliary_file( + RawPull *i, + const char *suffix, + char **temp_path, + char **path) { + + int r; + + assert(i); + assert(temp_path); + assert(suffix); + assert(path); + + /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should + * incorporate it in the file name if we can */ + *path = mfree(*path); + r = raw_pull_determine_path(i, suffix, path); + if (r < 0) + return r; + + r = import_make_read_only(*temp_path); + if (r < 0) + return r; + + r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path); + if (r < 0) + return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path); + + *temp_path = mfree(*temp_path); + + return 1; +} + +static void raw_pull_job_on_finished(PullJob *j) { + RawPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + if (j == i->roothash_job) { + if (j->error != 0) + log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without."); + } else if (j == i->settings_job) { + if (j->error != 0) + log_info_errno(j->error, "Settings file could not be retrieved, proceeding without."); + } else if (j->error != 0 && j != i->signature_job) { + if (j == i->checksum_job) + log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); + else + log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); + + r = j->error; + goto finish; + } + + /* This is invoked if either the download completed + * successfully, or the download was skipped because we + * already have the etag. In this case ->etag_exists is + * true. + * + * We only do something when we got all three files */ + + if (!raw_pull_is_done(i)) + return; + + if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) { + log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); + + r = i->signature_job->error; + goto finish; + } + + if (i->roothash_job) + i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd); + if (i->settings_job) + i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd); + + r = raw_pull_determine_path(i, ".raw", &i->final_path); + if (r < 0) + goto finish; + + if (!i->raw_job->etag_exists) { + /* This is a new download, verify it, and move it into place */ + assert(i->raw_job->disk_fd >= 0); + + raw_pull_report_progress(i, RAW_VERIFYING); + + r = pull_verify(i->raw_job, i->roothash_job, i->settings_job, i->checksum_job, i->signature_job); + if (r < 0) + goto finish; + + raw_pull_report_progress(i, RAW_UNPACKING); + + r = raw_pull_maybe_convert_qcow2(i); + if (r < 0) + goto finish; + + raw_pull_report_progress(i, RAW_FINALIZING); + + r = import_make_read_only_fd(i->raw_job->disk_fd); + if (r < 0) + goto finish; + + r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + if (r < 0) { + log_error_errno(r, "Failed to rename raw file to %s: %m", i->final_path); + goto finish; + } + + i->temp_path = mfree(i->temp_path); + + if (i->roothash_job && + i->roothash_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path); + if (r < 0) + goto finish; + } + + if (i->settings_job && + i->settings_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path); + if (r < 0) + goto finish; + } + } + + raw_pull_report_progress(i, RAW_COPYING); + + r = raw_pull_make_local_copy(i); + if (r < 0) + goto finish; + + r = 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); +} + +static int raw_pull_job_on_open_disk_generic( + RawPull *i, + PullJob *j, + const char *extra, + char **temp_path) { + + int r; + + assert(i); + assert(j); + assert(extra); + assert(temp_path); + + if (!*temp_path) { + r = tempfn_random_child(i->image_root, extra, temp_path); + if (r < 0) + return log_oom(); + } + + (void) mkdir_parents_label(*temp_path, 0700); + + j->disk_fd = open(*temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (j->disk_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", *temp_path); + + return 0; +} + +static int raw_pull_job_on_open_disk_raw(PullJob *j) { + RawPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->raw_job == j); + + r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path); + if (r < 0) + return r; + + r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL, NULL); + if (r < 0) + log_warning_errno(r, "Failed to set file attributes on %s, ignoring: %m", i->temp_path); + + return 0; +} + +static int raw_pull_job_on_open_disk_roothash(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->roothash_job == j); + + return raw_pull_job_on_open_disk_generic(i, j, "roothash", &i->roothash_temp_path); +} + +static int raw_pull_job_on_open_disk_settings(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->settings_job == j); + + return raw_pull_job_on_open_disk_generic(i, j, "settings", &i->settings_temp_path); +} + +static void raw_pull_job_on_progress(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + raw_pull_report_progress(i, RAW_DOWNLOADING); +} + +int raw_pull_start( + RawPull *i, + const char *url, + const char *local, + bool force_local, + ImportVerify verify, + bool settings, + bool roothash) { + + int r; + + assert(i); + assert(verify < _IMPORT_VERIFY_MAX); + assert(verify >= 0); + + if (!http_url_is_valid(url)) + return -EINVAL; + + if (local && !machine_name_is_valid(local)) + return -EINVAL; + + if (i->raw_job) + return -EBUSY; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + + i->force_local = force_local; + i->verify = verify; + i->settings = settings; + i->roothash = roothash; + + /* Queue job for the image itself */ + r = pull_job_new(&i->raw_job, url, i->glue, i); + if (r < 0) + return r; + + i->raw_job->on_finished = raw_pull_job_on_finished; + i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw; + i->raw_job->on_progress = raw_pull_job_on_progress; + i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO; + + r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags); + if (r < 0) + return r; + + if (roothash) { + r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i); + if (r < 0) + return r; + + i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash; + i->roothash_job->on_progress = raw_pull_job_on_progress; + i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO; + } + + if (settings) { + r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i); + if (r < 0) + return r; + + i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings; + i->settings_job->on_progress = raw_pull_job_on_progress; + i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO; + } + + r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i); + if (r < 0) + return r; + + r = pull_job_begin(i->raw_job); + if (r < 0) + return r; + + if (i->roothash_job) { + r = pull_job_begin(i->roothash_job); + if (r < 0) + return r; + } + + if (i->settings_job) { + r = pull_job_begin(i->settings_job); + if (r < 0) + return r; + } + + if (i->checksum_job) { + i->checksum_job->on_progress = raw_pull_job_on_progress; + i->checksum_job->style = VERIFICATION_PER_FILE; + + r = pull_job_begin(i->checksum_job); + if (r < 0) + return r; + } + + if (i->signature_job) { + i->signature_job->on_progress = raw_pull_job_on_progress; + + r = pull_job_begin(i->signature_job); + if (r < 0) + return r; + } + + return 0; +} diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h new file mode 100644 index 0000000..4ccd65b --- /dev/null +++ b/src/import/pull-raw.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include "sd-event.h" + +#include "import-util.h" +#include "macro.h" + +typedef struct RawPull RawPull; + +typedef void (*RawPullFinished)(RawPull *pull, int error, void *userdata); + +int raw_pull_new(RawPull **pull, sd_event *event, const char *image_root, RawPullFinished on_finished, void *userdata); +RawPull* raw_pull_unref(RawPull *pull); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref); + +int raw_pull_start(RawPull *pull, const char *url, const char *local, bool force_local, ImportVerify verify, bool settings, bool roothash); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c new file mode 100644 index 0000000..e7a208e --- /dev/null +++ b/src/import/pull-tar.c @@ -0,0 +1,559 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <curl/curl.h> +#include <sys/prctl.h> + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "curl-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-util.h" +#include "macro.h" +#include "mkdir.h" +#include "path-util.h" +#include "process-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "pull-tar.h" +#include "rm-rf.h" +#include "string-util.h" +#include "strv.h" +#include "tmpfile-util.h" +#include "utf8.h" +#include "util.h" +#include "web-util.h" + +typedef enum TarProgress { + TAR_DOWNLOADING, + TAR_VERIFYING, + TAR_FINALIZING, + TAR_COPYING, +} TarProgress; + +struct TarPull { + sd_event *event; + CurlGlue *glue; + + char *image_root; + + PullJob *tar_job; + PullJob *settings_job; + PullJob *checksum_job; + PullJob *signature_job; + + TarPullFinished on_finished; + void *userdata; + + char *local; + bool force_local; + bool settings; + + pid_t tar_pid; + + char *final_path; + char *temp_path; + + char *settings_path; + char *settings_temp_path; + + ImportVerify verify; +}; + +TarPull* tar_pull_unref(TarPull *i) { + if (!i) + return NULL; + + if (i->tar_pid > 1) { + (void) kill_and_sigcont(i->tar_pid, SIGKILL); + (void) wait_for_terminate(i->tar_pid, NULL); + } + + pull_job_unref(i->tar_job); + pull_job_unref(i->settings_job); + pull_job_unref(i->checksum_job); + pull_job_unref(i->signature_job); + + curl_glue_unref(i->glue); + sd_event_unref(i->event); + + if (i->temp_path) { + (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + free(i->temp_path); + } + + if (i->settings_temp_path) { + (void) unlink(i->settings_temp_path); + free(i->settings_temp_path); + } + + free(i->final_path); + free(i->settings_path); + free(i->image_root); + free(i->local); + + return mfree(i); +} + +int tar_pull_new( + TarPull **ret, + sd_event *event, + const char *image_root, + TarPullFinished on_finished, + void *userdata) { + + _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL; + _cleanup_(sd_event_unrefp) sd_event *e = NULL; + _cleanup_(tar_pull_unrefp) TarPull *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + if (event) + e = sd_event_ref(event); + else { + r = sd_event_default(&e); + if (r < 0) + return r; + } + + r = curl_glue_new(&g, e); + if (r < 0) + return r; + + i = new(TarPull, 1); + if (!i) + return -ENOMEM; + + *i = (TarPull) { + .on_finished = on_finished, + .userdata = userdata, + .image_root = TAKE_PTR(root), + .event = TAKE_PTR(e), + .glue = TAKE_PTR(g), + }; + + i->glue->on_finished = pull_job_curl_on_finished; + i->glue->userdata = i; + + *ret = TAKE_PTR(i); + + return 0; +} + +static void tar_pull_report_progress(TarPull *i, TarProgress p) { + unsigned percent; + + assert(i); + + switch (p) { + + case TAR_DOWNLOADING: { + unsigned remain = 85; + + percent = 0; + + if (i->settings_job) { + percent += i->settings_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->checksum_job) { + percent += i->checksum_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->signature_job) { + percent += i->signature_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->tar_job) + percent += i->tar_job->progress_percent * remain / 100; + break; + } + + case TAR_VERIFYING: + percent = 85; + break; + + case TAR_FINALIZING: + percent = 90; + break; + + case TAR_COPYING: + percent = 95; + break; + + default: + assert_not_reached("Unknown progress state"); + } + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_debug("Combined progress %u%%", percent); +} + +static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) { + int r; + + assert(i); + assert(field); + + if (*field) + return 0; + + assert(i->tar_job); + + r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field); + if (r < 0) + return log_oom(); + + return 1; +} + +static int tar_pull_make_local_copy(TarPull *i) { + int r; + + assert(i); + assert(i->tar_job); + + if (!i->local) + return 0; + + r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local); + if (r < 0) + return r; + + if (i->settings) { + const char *local_settings; + assert(i->settings_job); + + r = tar_pull_determine_path(i, ".nspawn", &i->settings_path); + if (r < 0) + return r; + + local_settings = strjoina(i->image_root, "/", i->local, ".nspawn"); + + r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0)); + if (r == -EEXIST) + log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings); + else if (r == -ENOENT) + log_debug_errno(r, "Skipping creation of settings file, since none was found."); + else if (r < 0) + log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings); + else + log_info("Created new settings file %s.", local_settings); + } + + return 0; +} + +static bool tar_pull_is_done(TarPull *i) { + assert(i); + assert(i->tar_job); + + if (!PULL_JOB_IS_COMPLETE(i->tar_job)) + return false; + if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job)) + return false; + if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job)) + return false; + if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job)) + return false; + + return true; +} + +static void tar_pull_job_on_finished(PullJob *j) { + TarPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + if (j == i->settings_job) { + if (j->error != 0) + log_info_errno(j->error, "Settings file could not be retrieved, proceeding without."); + } else if (j->error != 0 && j != i->signature_job) { + if (j == i->checksum_job) + log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); + else + log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); + + r = j->error; + goto finish; + } + + /* This is invoked if either the download completed + * successfully, or the download was skipped because we + * already have the etag. */ + + if (!tar_pull_is_done(i)) + return; + + if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) { + log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); + + r = i->signature_job->error; + goto finish; + } + + i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd); + if (i->settings_job) + i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd); + + r = tar_pull_determine_path(i, NULL, &i->final_path); + if (r < 0) + goto finish; + + if (i->tar_pid > 0) { + r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG); + i->tar_pid = 0; + if (r < 0) + goto finish; + if (r != EXIT_SUCCESS) { + r = -EIO; + goto finish; + } + } + + if (!i->tar_job->etag_exists) { + /* This is a new download, verify it, and move it into place */ + + tar_pull_report_progress(i, TAR_VERIFYING); + + r = pull_verify(i->tar_job, NULL, i->settings_job, i->checksum_job, i->signature_job); + if (r < 0) + goto finish; + + tar_pull_report_progress(i, TAR_FINALIZING); + + r = import_make_read_only(i->temp_path); + if (r < 0) + goto finish; + + r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + if (r < 0) { + log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path); + goto finish; + } + + i->temp_path = mfree(i->temp_path); + + if (i->settings_job && + i->settings_job->error == 0) { + + /* Also move the settings file into place, if it exists. Note that we do so only if we also + * moved the tar file in place, to keep things strictly in sync. */ + assert(i->settings_temp_path); + + /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and + * we should incorporate it in the file name if we can */ + i->settings_path = mfree(i->settings_path); + + r = tar_pull_determine_path(i, ".nspawn", &i->settings_path); + if (r < 0) + goto finish; + + r = import_make_read_only(i->settings_temp_path); + if (r < 0) + goto finish; + + r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path); + if (r < 0) { + log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path); + goto finish; + } + + i->settings_temp_path = mfree(i->settings_temp_path); + } + } + + tar_pull_report_progress(i, TAR_COPYING); + + r = tar_pull_make_local_copy(i); + if (r < 0) + goto finish; + + r = 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); +} + +static int tar_pull_job_on_open_disk_tar(PullJob *j) { + TarPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->tar_job == j); + assert(i->tar_pid <= 0); + + if (!i->temp_path) { + r = tempfn_random_child(i->image_root, "tar", &i->temp_path); + if (r < 0) + return log_oom(); + } + + mkdir_parents_label(i->temp_path, 0700); + + r = btrfs_subvol_make(i->temp_path); + if (r == -ENOTTY) { + if (mkdir(i->temp_path, 0755) < 0) + return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path); + } else if (r < 0) + return log_error_errno(r, "Failed to create subvolume %s: %m", i->temp_path); + else + (void) import_assign_pool_quota_and_warn(i->temp_path); + + j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); + if (j->disk_fd < 0) + return j->disk_fd; + + return 0; +} + +static int tar_pull_job_on_open_disk_settings(PullJob *j) { + TarPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->settings_job == j); + + if (!i->settings_temp_path) { + r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path); + if (r < 0) + return log_oom(); + } + + mkdir_parents_label(i->settings_temp_path, 0700); + + j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (j->disk_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path); + + return 0; +} + +static void tar_pull_job_on_progress(PullJob *j) { + TarPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + tar_pull_report_progress(i, TAR_DOWNLOADING); +} + +int tar_pull_start( + TarPull *i, + const char *url, + const char *local, + bool force_local, + ImportVerify verify, + bool settings) { + + int r; + + assert(i); + assert(verify < _IMPORT_VERIFY_MAX); + assert(verify >= 0); + + if (!http_url_is_valid(url)) + return -EINVAL; + + if (local && !machine_name_is_valid(local)) + return -EINVAL; + + if (i->tar_job) + return -EBUSY; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + + i->force_local = force_local; + i->verify = verify; + i->settings = settings; + + /* Set up download job for TAR file */ + r = pull_job_new(&i->tar_job, url, i->glue, i); + if (r < 0) + return r; + + i->tar_job->on_finished = tar_pull_job_on_finished; + i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar; + i->tar_job->on_progress = tar_pull_job_on_progress; + i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO; + + r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags); + if (r < 0) + return r; + + /* Set up download job for the settings file (.nspawn) */ + if (settings) { + r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i); + if (r < 0) + return r; + + i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings; + i->settings_job->on_progress = tar_pull_job_on_progress; + i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO; + } + + /* Set up download of checksum/signature files */ + r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i); + if (r < 0) + return r; + + r = pull_job_begin(i->tar_job); + if (r < 0) + return r; + + if (i->settings_job) { + r = pull_job_begin(i->settings_job); + if (r < 0) + return r; + } + + if (i->checksum_job) { + i->checksum_job->on_progress = tar_pull_job_on_progress; + i->checksum_job->style = VERIFICATION_PER_FILE; + + r = pull_job_begin(i->checksum_job); + if (r < 0) + return r; + } + + if (i->signature_job) { + i->signature_job->on_progress = tar_pull_job_on_progress; + + r = pull_job_begin(i->signature_job); + if (r < 0) + return r; + } + + return 0; +} diff --git a/src/import/pull-tar.h b/src/import/pull-tar.h new file mode 100644 index 0000000..76f920e --- /dev/null +++ b/src/import/pull-tar.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include "sd-event.h" + +#include "import-util.h" +#include "macro.h" + +typedef struct TarPull TarPull; + +typedef void (*TarPullFinished)(TarPull *pull, int error, void *userdata); + +int tar_pull_new(TarPull **pull, sd_event *event, const char *image_root, TarPullFinished on_finished, void *userdata); +TarPull* tar_pull_unref(TarPull *pull); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref); + +int tar_pull_start(TarPull *pull, const char *url, const char *local, bool force_local, ImportVerify verify, bool settings); diff --git a/src/import/pull.c b/src/import/pull.c new file mode 100644 index 0000000..3376992 --- /dev/null +++ b/src/import/pull.c @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <getopt.h> + +#include "sd-event.h" +#include "sd-id128.h" + +#include "alloc-util.h" +#include "hostname-util.h" +#include "import-util.h" +#include "machine-image.h" +#include "main-func.h" +#include "parse-util.h" +#include "pull-raw.h" +#include "pull-tar.h" +#include "signal-util.h" +#include "string-util.h" +#include "verbs.h" +#include "web-util.h" + +static bool arg_force = false; +static const char *arg_image_root = "/var/lib/machines"; +static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; +static bool arg_settings = true; +static bool arg_roothash = true; + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +static void on_tar_finished(TarPull *pull, int error, void *userdata) { + sd_event *event = userdata; + assert(pull); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int pull_tar(int argc, char *argv[], void *userdata) { + _cleanup_(tar_pull_unrefp) TarPull *pull = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + const char *url, *local; + _cleanup_free_ char *l = NULL, *ll = NULL; + int r; + + url = argv[1]; + if (!http_url_is_valid(url)) { + log_error("URL '%s' is not valid.", url); + return -EINVAL; + } + + if (argc >= 3) + local = argv[2]; + else { + r = import_url_last_component(url, &l); + if (r < 0) + return log_error_errno(r, "Failed get final component of URL: %m"); + + local = l; + } + + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (local) { + r = tar_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local image name '%s' is not valid.", local); + return -EINVAL; + } + + if (!arg_force) { + r = image_find(IMAGE_MACHINE, local, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else { + log_error("Image '%s' already exists.", local); + return -EEXIST; + } + } + + log_info("Pulling '%s', saving as '%s'.", url, local); + } else + log_info("Pulling '%s'.", url); + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate puller: %m"); + + r = tar_pull_start(pull, url, local, arg_force, arg_verify, arg_settings); + if (r < 0) + return log_error_errno(r, "Failed to pull image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawPull *pull, int error, void *userdata) { + sd_event *event = userdata; + assert(pull); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int pull_raw(int argc, char *argv[], void *userdata) { + _cleanup_(raw_pull_unrefp) RawPull *pull = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + const char *url, *local; + _cleanup_free_ char *l = NULL, *ll = NULL; + int r; + + url = argv[1]; + if (!http_url_is_valid(url)) { + log_error("URL '%s' is not valid.", url); + return -EINVAL; + } + + if (argc >= 3) + local = argv[2]; + else { + r = import_url_last_component(url, &l); + if (r < 0) + return log_error_errno(r, "Failed get final component of URL: %m"); + + local = l; + } + + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (local) { + r = raw_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local image name '%s' is not valid.", local); + return -EINVAL; + } + + if (!arg_force) { + r = image_find(IMAGE_MACHINE, local, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else { + log_error("Image '%s' already exists.", local); + return -EEXIST; + } + } + + log_info("Pulling '%s', saving as '%s'.", url, local); + } else + log_info("Pulling '%s'.", url); + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate puller: %m"); + + r = raw_pull_start(pull, url, local, arg_force, arg_verify, arg_settings, arg_roothash); + if (r < 0) + return log_error_errno(r, "Failed to pull image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%s [OPTIONS...] {COMMAND} ...\n\n" + "Download container or virtual machine images.\n\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --force Force creation of image\n" + " --verify=MODE Verify downloaded image, one of: 'no',\n" + " 'checksum', 'signature'\n" + " --settings=BOOL Download settings file with image\n" + " --roothash=BOOL Download root hash file with image\n" + " --image-root=PATH Image root directory\n\n" + "Commands:\n" + " tar URL [NAME] Download a TAR image\n" + " raw URL [NAME] Download a RAW image\n", + program_invocation_short_name); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORCE, + ARG_IMAGE_ROOT, + ARG_VERIFY, + ARG_SETTINGS, + ARG_ROOTHASH, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "force", no_argument, NULL, ARG_FORCE }, + { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, + { "verify", required_argument, NULL, ARG_VERIFY }, + { "settings", required_argument, NULL, ARG_SETTINGS }, + { "roothash", required_argument, NULL, ARG_ROOTHASH }, + {} + }; + + int c, r; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORCE: + arg_force = true; + break; + + case ARG_IMAGE_ROOT: + arg_image_root = optarg; + break; + + case ARG_VERIFY: + arg_verify = import_verify_from_string(optarg); + if (arg_verify < 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Invalid verification setting '%s'", optarg); + + break; + + case ARG_SETTINGS: + r = parse_boolean(optarg); + if (r < 0) + return log_error_errno(r, "Failed to parse --settings= parameter '%s': %m", optarg); + + arg_settings = r; + break; + + case ARG_ROOTHASH: + r = parse_boolean(optarg); + if (r < 0) + return log_error_errno(r, "Failed to parse --roothash= parameter '%s': %m", optarg); + + arg_roothash = r; + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached("Unhandled option"); + } + + return 1; +} + +static int pull_main(int argc, char *argv[]) { + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, pull_tar }, + { "raw", 2, 3, 0, pull_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +static int run(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + return r; + + (void) ignore_signals(SIGPIPE, -1); + + return pull_main(argc, argv); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/qcow2-util.c b/src/import/qcow2-util.c new file mode 100644 index 0000000..e927b60 --- /dev/null +++ b/src/import/qcow2-util.c @@ -0,0 +1,334 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include <zlib.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "qcow2-util.h" +#include "sparse-endian.h" +#include "util.h" + +#define QCOW2_MAGIC 0x514649fb + +#define QCOW2_COPIED (1ULL << 63) +#define QCOW2_COMPRESSED (1ULL << 62) +#define QCOW2_ZERO (1ULL << 0) + +typedef struct _packed_ Header { + be32_t magic; + be32_t version; + + be64_t backing_file_offset; + be32_t backing_file_size; + + be32_t cluster_bits; + be64_t size; + be32_t crypt_method; + + be32_t l1_size; + be64_t l1_table_offset; + + be64_t refcount_table_offset; + be32_t refcount_table_clusters; + + be32_t nb_snapshots; + be64_t snapshots_offset; + + /* The remainder is only present on QCOW3 */ + be64_t incompatible_features; + be64_t compatible_features; + be64_t autoclear_features; + + be32_t refcount_order; + be32_t header_length; +} Header; + +#define HEADER_MAGIC(header) be32toh((header)->magic) +#define HEADER_VERSION(header) be32toh((header)->version) +#define HEADER_CLUSTER_BITS(header) be32toh((header)->cluster_bits) +#define HEADER_CLUSTER_SIZE(header) (1ULL << HEADER_CLUSTER_BITS(header)) +#define HEADER_L2_BITS(header) (HEADER_CLUSTER_BITS(header) - 3) +#define HEADER_SIZE(header) be64toh((header)->size) +#define HEADER_CRYPT_METHOD(header) be32toh((header)->crypt_method) +#define HEADER_L1_SIZE(header) be32toh((header)->l1_size) +#define HEADER_L2_SIZE(header) (HEADER_CLUSTER_SIZE(header)/sizeof(uint64_t)) +#define HEADER_L1_TABLE_OFFSET(header) be64toh((header)->l1_table_offset) + +static uint32_t HEADER_HEADER_LENGTH(const Header *h) { + if (HEADER_VERSION(h) < 3) + return offsetof(Header, incompatible_features); + + return be32toh(h->header_length); +} + +static int copy_cluster( + int sfd, uint64_t soffset, + int dfd, uint64_t doffset, + uint64_t cluster_size, + void *buffer) { + + ssize_t l; + int r; + + r = btrfs_clone_range(sfd, soffset, dfd, doffset, cluster_size); + if (r >= 0) + return r; + + l = pread(sfd, buffer, cluster_size, soffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + l = pwrite(dfd, buffer, cluster_size, doffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + return 0; +} + +static int decompress_cluster( + int sfd, uint64_t soffset, + int dfd, uint64_t doffset, + uint64_t compressed_size, + uint64_t cluster_size, + void *buffer1, + void *buffer2) { + + _cleanup_free_ void *large_buffer = NULL; + z_stream s = {}; + uint64_t sz; + ssize_t l; + int r; + + if (compressed_size > cluster_size) { + /* The usual cluster buffer doesn't suffice, let's + * allocate a larger one, temporarily */ + + large_buffer = malloc(compressed_size); + if (!large_buffer) + return -ENOMEM; + + buffer1 = large_buffer; + } + + l = pread(sfd, buffer1, compressed_size, soffset); + if (l < 0) + return -errno; + if ((uint64_t) l != compressed_size) + return -EIO; + + s.next_in = buffer1; + s.avail_in = compressed_size; + s.next_out = buffer2; + s.avail_out = cluster_size; + + r = inflateInit2(&s, -12); + if (r != Z_OK) + return -EIO; + + r = inflate(&s, Z_FINISH); + sz = (uint8_t*) s.next_out - (uint8_t*) buffer2; + inflateEnd(&s); + if (r != Z_STREAM_END || sz != cluster_size) + return -EIO; + + l = pwrite(dfd, buffer2, cluster_size, doffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + return 0; +} + +static int normalize_offset( + const Header *header, + uint64_t p, + uint64_t *ret, + bool *compressed, + uint64_t *compressed_size) { + + uint64_t q; + + q = be64toh(p); + + if (q & QCOW2_COMPRESSED) { + uint64_t sz, csize_shift, csize_mask; + + if (!compressed) + return -EOPNOTSUPP; + + csize_shift = 64 - 2 - (HEADER_CLUSTER_BITS(header) - 8); + csize_mask = (1ULL << (HEADER_CLUSTER_BITS(header) - 8)) - 1; + sz = (((q >> csize_shift) & csize_mask) + 1) * 512 - (q & 511); + q &= ((1ULL << csize_shift) - 1); + + if (compressed_size) + *compressed_size = sz; + + *compressed = true; + + } else { + if (compressed) { + *compressed = false; + *compressed_size = 0; + } + + if (q & QCOW2_ZERO) { + /* We make no distinction between zero blocks and holes */ + *ret = 0; + return 0; + } + + q &= ~QCOW2_COPIED; + } + + *ret = q; + return q > 0; /* returns positive if not a hole */ +} + +static int verify_header(const Header *header) { + assert(header); + + if (HEADER_MAGIC(header) != QCOW2_MAGIC) + return -EBADMSG; + + if (!IN_SET(HEADER_VERSION(header), 2, 3)) + return -EOPNOTSUPP; + + if (HEADER_CRYPT_METHOD(header) != 0) + return -EOPNOTSUPP; + + if (HEADER_CLUSTER_BITS(header) < 9) /* 512K */ + return -EBADMSG; + + if (HEADER_CLUSTER_BITS(header) > 21) /* 2MB */ + return -EBADMSG; + + if (HEADER_SIZE(header) % HEADER_CLUSTER_SIZE(header) != 0) + return -EBADMSG; + + if (HEADER_L1_SIZE(header) > 32*1024*1024) /* 32MB */ + return -EBADMSG; + + if (HEADER_VERSION(header) == 3) { + + if (header->incompatible_features != 0) + return -EOPNOTSUPP; + + if (HEADER_HEADER_LENGTH(header) < sizeof(Header)) + return -EBADMSG; + } + + return 0; +} + +int qcow2_convert(int qcow2_fd, int raw_fd) { + _cleanup_free_ void *buffer1 = NULL, *buffer2 = NULL; + _cleanup_free_ be64_t *l1_table = NULL, *l2_table = NULL; + uint64_t sz, i; + Header header; + ssize_t l; + int r; + + l = pread(qcow2_fd, &header, sizeof(header), 0); + if (l < 0) + return -errno; + if (l != sizeof(header)) + return -EIO; + + r = verify_header(&header); + if (r < 0) + return r; + + l1_table = new(be64_t, HEADER_L1_SIZE(&header)); + if (!l1_table) + return -ENOMEM; + + l2_table = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!l2_table) + return -ENOMEM; + + buffer1 = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!buffer1) + return -ENOMEM; + + buffer2 = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!buffer2) + return -ENOMEM; + + /* Empty the file if it exists, we rely on zero bits */ + if (ftruncate(raw_fd, 0) < 0) + return -errno; + + if (ftruncate(raw_fd, HEADER_SIZE(&header)) < 0) + return -errno; + + sz = sizeof(uint64_t) * HEADER_L1_SIZE(&header); + l = pread(qcow2_fd, l1_table, sz, HEADER_L1_TABLE_OFFSET(&header)); + if (l < 0) + return -errno; + if ((uint64_t) l != sz) + return -EIO; + + for (i = 0; i < HEADER_L1_SIZE(&header); i ++) { + uint64_t l2_begin, j; + + r = normalize_offset(&header, l1_table[i], &l2_begin, NULL, NULL); + if (r < 0) + return r; + if (r == 0) + continue; + + l = pread(qcow2_fd, l2_table, HEADER_CLUSTER_SIZE(&header), l2_begin); + if (l < 0) + return -errno; + if ((uint64_t) l != HEADER_CLUSTER_SIZE(&header)) + return -EIO; + + for (j = 0; j < HEADER_L2_SIZE(&header); j++) { + uint64_t data_begin, p, compressed_size; + bool compressed; + + p = ((i << HEADER_L2_BITS(&header)) + j) << HEADER_CLUSTER_BITS(&header); + + r = normalize_offset(&header, l2_table[j], &data_begin, &compressed, &compressed_size); + if (r < 0) + return r; + if (r == 0) + continue; + + if (compressed) + r = decompress_cluster( + qcow2_fd, data_begin, + raw_fd, p, + compressed_size, HEADER_CLUSTER_SIZE(&header), + buffer1, buffer2); + else + r = copy_cluster( + qcow2_fd, data_begin, + raw_fd, p, + HEADER_CLUSTER_SIZE(&header), buffer1); + if (r < 0) + return r; + } + } + + return 0; +} + +int qcow2_detect(int fd) { + be32_t id; + ssize_t l; + + l = pread(fd, &id, sizeof(id), 0); + if (l < 0) + return -errno; + if (l != sizeof(id)) + return -EIO; + + return htobe32(QCOW2_MAGIC) == id; +} diff --git a/src/import/qcow2-util.h b/src/import/qcow2-util.h new file mode 100644 index 0000000..7393d98 --- /dev/null +++ b/src/import/qcow2-util.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +int qcow2_detect(int fd); +int qcow2_convert(int qcow2_fd, int raw_fd); diff --git a/src/import/test-qcow2.c b/src/import/test-qcow2.c new file mode 100644 index 0000000..bd2b458 --- /dev/null +++ b/src/import/test-qcow2.c @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include "fd-util.h" +#include "log.h" +#include "qcow2-util.h" +#include "util.h" + +int main(int argc, char *argv[]) { + _cleanup_close_ int sfd = -1, dfd = -1; + int r; + + if (argc != 3) { + log_error("Needs two arguments."); + return EXIT_FAILURE; + } + + sfd = open(argv[1], O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (sfd < 0) { + log_error_errno(errno, "Can't open source file: %m"); + return EXIT_FAILURE; + } + + dfd = open(argv[2], O_WRONLY|O_CREAT|O_CLOEXEC|O_NOCTTY, 0666); + if (dfd < 0) { + log_error_errno(errno, "Can't open destination file: %m"); + return EXIT_FAILURE; + } + + r = qcow2_convert(sfd, dfd); + if (r < 0) { + log_error_errno(r, "Failed to unpack: %m"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} |