diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 15:35:18 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 15:35:18 +0000 |
commit | b750101eb236130cf056c675997decbac904cc49 (patch) | |
tree | a5df1a06754bdd014cb975c051c83b01c9a97532 /src/import | |
parent | Initial commit. (diff) | |
download | systemd-b750101eb236130cf056c675997decbac904cc49.tar.xz systemd-b750101eb236130cf056c675997decbac904cc49.zip |
Adding upstream version 252.22.upstream/252.22
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/import')
35 files changed, 9986 insertions, 0 deletions
diff --git a/src/import/curl-util.c b/src/import/curl-util.c new file mode 100644 index 0000000..94f718d --- /dev/null +++ b/src/import/curl-util.c @@ -0,0 +1,384 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <fcntl.h> + +#include "alloc-util.h" +#include "curl-util.h" +#include "fd-util.h" +#include "locale-util.h" +#include "string-util.h" +#include "version.h" + +static void curl_glue_check_finished(CurlGlue *g) { + CURLMsg *msg; + int k = 0; + + assert(g); + + msg = curl_multi_info_read(g->curl, &k); + if (!msg) + return; + + if (msg->msg != CURLMSG_DONE) + return; + + if (g->on_finished) + g->on_finished(g, msg->easy_handle, msg->data.result); +} + +static int curl_glue_on_io(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + CurlGlue *g = ASSERT_PTR(userdata); + int action, k = 0; + + assert(s); + + if (FLAGS_SET(revents, EPOLLIN | EPOLLOUT)) + action = CURL_POLL_INOUT; + else if (revents & EPOLLIN) + action = CURL_POLL_IN; + else if (revents & EPOLLOUT) + action = CURL_POLL_OUT; + else + action = 0; + + if (curl_multi_socket_action(g->curl, fd, action, &k) != CURLM_OK) + return log_debug_errno(SYNTHETIC_ERRNO(EINVAL), + "Failed to propagate IO event."); + + curl_glue_check_finished(g); + return 0; +} + +static int curl_glue_socket_callback(CURL *curl, curl_socket_t s, int action, void *userdata, void *socketp) { + sd_event_source *io = socketp; + CurlGlue *g = ASSERT_PTR(userdata); + uint32_t events = 0; + int r; + + assert(curl); + + if (action == CURL_POLL_REMOVE) { + if (io) { + sd_event_source_disable_unref(io); + + hashmap_remove(g->ios, FD_TO_PTR(s)); + } + + return 0; + } + + r = hashmap_ensure_allocated(&g->ios, &trivial_hash_ops); + if (r < 0) { + log_oom(); + return -1; + } + + if (action == CURL_POLL_IN) + events = EPOLLIN; + else if (action == CURL_POLL_OUT) + events = EPOLLOUT; + else if (action == CURL_POLL_INOUT) + events = EPOLLIN|EPOLLOUT; + + if (io) { + if (sd_event_source_set_io_events(io, events) < 0) + return -1; + + if (sd_event_source_set_enabled(io, SD_EVENT_ON) < 0) + return -1; + } else { + if (sd_event_add_io(g->event, &io, s, events, curl_glue_on_io, g) < 0) + return -1; + + if (curl_multi_assign(g->curl, s, io) != CURLM_OK) + return -1; + + (void) sd_event_source_set_description(io, "curl-io"); + + r = hashmap_put(g->ios, FD_TO_PTR(s), io); + if (r < 0) { + log_oom(); + sd_event_source_unref(io); + return -1; + } + } + + return 0; +} + +static int curl_glue_on_timer(sd_event_source *s, uint64_t usec, void *userdata) { + CurlGlue *g = ASSERT_PTR(userdata); + int k = 0; + + assert(s); + + if (curl_multi_socket_action(g->curl, CURL_SOCKET_TIMEOUT, 0, &k) != CURLM_OK) + return log_debug_errno(SYNTHETIC_ERRNO(EINVAL), + "Failed to propagate timeout."); + + curl_glue_check_finished(g); + return 0; +} + +static int curl_glue_timer_callback(CURLM *curl, long timeout_ms, void *userdata) { + CurlGlue *g = ASSERT_PTR(userdata); + usec_t usec; + + assert(curl); + + if (timeout_ms < 0) { + if (g->timer) { + if (sd_event_source_set_enabled(g->timer, SD_EVENT_OFF) < 0) + return -1; + } + + return 0; + } + + usec = (usec_t) timeout_ms * USEC_PER_MSEC + USEC_PER_MSEC - 1; + + if (g->timer) { + if (sd_event_source_set_time_relative(g->timer, usec) < 0) + return -1; + + if (sd_event_source_set_enabled(g->timer, SD_EVENT_ONESHOT) < 0) + return -1; + } else { + if (sd_event_add_time_relative(g->event, &g->timer, CLOCK_BOOTTIME, usec, 0, curl_glue_on_timer, g) < 0) + return -1; + + (void) sd_event_source_set_description(g->timer, "curl-timer"); + } + + return 0; +} + +CurlGlue *curl_glue_unref(CurlGlue *g) { + sd_event_source *io; + + if (!g) + return NULL; + + if (g->curl) + curl_multi_cleanup(g->curl); + + while ((io = hashmap_steal_first(g->ios))) + sd_event_source_unref(io); + + hashmap_free(g->ios); + + sd_event_source_unref(g->timer); + sd_event_unref(g->event); + return mfree(g); +} + +int curl_glue_new(CurlGlue **glue, sd_event *event) { + _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL; + _cleanup_(curl_multi_cleanupp) CURLM *c = NULL; + _cleanup_(sd_event_unrefp) sd_event *e = NULL; + int r; + + if (event) + e = sd_event_ref(event); + else { + r = sd_event_default(&e); + if (r < 0) + return r; + } + + c = curl_multi_init(); + if (!c) + return -ENOMEM; + + g = new(CurlGlue, 1); + if (!g) + return -ENOMEM; + + *g = (CurlGlue) { + .event = TAKE_PTR(e), + .curl = TAKE_PTR(c), + }; + + if (curl_multi_setopt(g->curl, CURLMOPT_SOCKETDATA, g) != CURLM_OK) + return -EINVAL; + + if (curl_multi_setopt(g->curl, CURLMOPT_SOCKETFUNCTION, curl_glue_socket_callback) != CURLM_OK) + return -EINVAL; + + if (curl_multi_setopt(g->curl, CURLMOPT_TIMERDATA, g) != CURLM_OK) + return -EINVAL; + + if (curl_multi_setopt(g->curl, CURLMOPT_TIMERFUNCTION, curl_glue_timer_callback) != CURLM_OK) + return -EINVAL; + + *glue = TAKE_PTR(g); + + return 0; +} + +int curl_glue_make(CURL **ret, const char *url, void *userdata) { + _cleanup_(curl_easy_cleanupp) CURL *c = NULL; + const char *useragent; + + assert(ret); + assert(url); + + c = curl_easy_init(); + if (!c) + return -ENOMEM; + + if (DEBUG_LOGGING) + (void) curl_easy_setopt(c, CURLOPT_VERBOSE, 1L); + + if (curl_easy_setopt(c, CURLOPT_URL, url) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(c, CURLOPT_PRIVATE, userdata) != CURLE_OK) + return -EIO; + + useragent = strjoina(program_invocation_short_name, "/" GIT_VERSION); + if (curl_easy_setopt(c, CURLOPT_USERAGENT, useragent) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(c, CURLOPT_LOW_SPEED_TIME, 60L) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(c, CURLOPT_LOW_SPEED_LIMIT, 30L) != CURLE_OK) + return -EIO; + +#if LIBCURL_VERSION_NUM >= 0x075500 /* libcurl 7.85.0 */ + if (curl_easy_setopt(c, CURLOPT_PROTOCOLS_STR, "HTTP,HTTPS,FILE") != CURLE_OK) +#else + if (curl_easy_setopt(c, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS|CURLPROTO_FILE) != CURLE_OK) +#endif + return -EIO; + + *ret = TAKE_PTR(c); + return 0; +} + +int curl_glue_add(CurlGlue *g, CURL *c) { + assert(g); + assert(c); + + if (curl_multi_add_handle(g->curl, c) != CURLM_OK) + return -EIO; + + return 0; +} + +void curl_glue_remove_and_free(CurlGlue *g, CURL *c) { + assert(g); + + if (!c) + return; + + if (g->curl) + curl_multi_remove_handle(g->curl, c); + + curl_easy_cleanup(c); +} + +struct curl_slist *curl_slist_new(const char *first, ...) { + struct curl_slist *l; + va_list ap; + + if (!first) + return NULL; + + l = curl_slist_append(NULL, first); + if (!l) + return NULL; + + va_start(ap, first); + + for (;;) { + struct curl_slist *n; + const char *i; + + i = va_arg(ap, const char*); + if (!i) + break; + + n = curl_slist_append(l, i); + if (!n) { + va_end(ap); + curl_slist_free_all(l); + return NULL; + } + + l = n; + } + + va_end(ap); + return l; +} + +int curl_header_strdup(const void *contents, size_t sz, const char *field, char **value) { + const char *p; + char *s; + + p = memory_startswith_no_case(contents, sz, field); + if (!p) + return 0; + + sz -= p - (const char*) contents; + + if (memchr(p, 0, sz)) + return 0; + + /* Skip over preceding whitespace */ + while (sz > 0 && strchr(WHITESPACE, p[0])) { + p++; + sz--; + } + + /* Truncate trailing whitespace */ + while (sz > 0 && strchr(WHITESPACE, p[sz-1])) + sz--; + + s = strndup(p, sz); + if (!s) + return -ENOMEM; + + *value = s; + return 1; +} + +int curl_parse_http_time(const char *t, usec_t *ret) { + _cleanup_(freelocalep) locale_t loc = (locale_t) 0; + const char *e; + struct tm tm; + time_t v; + + assert(t); + assert(ret); + + loc = newlocale(LC_TIME_MASK, "C", (locale_t) 0); + if (loc == (locale_t) 0) + return -errno; + + /* RFC822 */ + e = strptime_l(t, "%a, %d %b %Y %H:%M:%S %Z", &tm, loc); + if (!e || *e != 0) + /* RFC 850 */ + e = strptime_l(t, "%A, %d-%b-%y %H:%M:%S %Z", &tm, loc); + if (!e || *e != 0) + /* ANSI C */ + e = strptime_l(t, "%a %b %d %H:%M:%S %Y", &tm, loc); + if (!e || *e != 0) + return -EINVAL; + + v = timegm(&tm); + if (v == (time_t) -1) + return -EINVAL; + + *ret = (usec_t) v * USEC_PER_SEC; + return 0; +} diff --git a/src/import/curl-util.h b/src/import/curl-util.h new file mode 100644 index 0000000..6b4f992 --- /dev/null +++ b/src/import/curl-util.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include <curl/curl.h> +#include <sys/types.h> + +#include "sd-event.h" + +#include "hashmap.h" +#include "time-util.h" + +typedef struct CurlGlue CurlGlue; + +struct CurlGlue { + sd_event *event; + CURLM *curl; + sd_event_source *timer; + Hashmap *ios; + + void (*on_finished)(CurlGlue *g, CURL *curl, CURLcode code); + void *userdata; +}; + +int curl_glue_new(CurlGlue **glue, sd_event *event); +CurlGlue* curl_glue_unref(CurlGlue *glue); + +DEFINE_TRIVIAL_CLEANUP_FUNC(CurlGlue*, curl_glue_unref); + +int curl_glue_make(CURL **ret, const char *url, void *userdata); +int curl_glue_add(CurlGlue *g, CURL *c); +void curl_glue_remove_and_free(CurlGlue *g, CURL *c); + +struct curl_slist *curl_slist_new(const char *first, ...) _sentinel_; +int curl_header_strdup(const void *contents, size_t sz, const char *field, char **value); +int curl_parse_http_time(const char *t, usec_t *ret); + +DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(CURL*, curl_easy_cleanup, NULL); +DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(CURLM*, curl_multi_cleanup, NULL); +DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(struct curl_slist*, curl_slist_free_all, NULL); diff --git a/src/import/export-raw.c b/src/import/export-raw.c new file mode 100644 index 0000000..a3ff6a3 --- /dev/null +++ b/src/import/export-raw.c @@ -0,0 +1,325 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <sys/sendfile.h> + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "export-raw.h" +#include "fd-util.h" +#include "fs-util.h" +#include "import-common.h" +#include "missing_fcntl.h" +#include "ratelimit.h" +#include "stat-util.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +#define COPY_BUFFER_SIZE (16*1024) + +struct RawExport { + sd_event *event; + + RawExportFinished on_finished; + void *userdata; + + char *path; + + int input_fd; + int output_fd; + + ImportCompress compress; + + sd_event_source *output_event_source; + + void *buffer; + size_t buffer_size; + size_t buffer_allocated; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + unsigned last_percent; + RateLimit progress_ratelimit; + + struct stat st; + + bool eof; + bool tried_reflink; + bool tried_sendfile; +}; + +RawExport *raw_export_unref(RawExport *e) { + if (!e) + return NULL; + + sd_event_source_unref(e->output_event_source); + + import_compress_free(&e->compress); + + sd_event_unref(e->event); + + safe_close(e->input_fd); + + free(e->buffer); + free(e->path); + return mfree(e); +} + +int raw_export_new( + RawExport **ret, + sd_event *event, + RawExportFinished on_finished, + void *userdata) { + + _cleanup_(raw_export_unrefp) RawExport *e = NULL; + int r; + + assert(ret); + + e = new(RawExport, 1); + if (!e) + return -ENOMEM; + + *e = (RawExport) { + .output_fd = -1, + .input_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .last_percent = UINT_MAX, + .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 }, + }; + + if (event) + e->event = sd_event_ref(event); + else { + r = sd_event_default(&e->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(e); + + return 0; +} + +static void raw_export_report_progress(RawExport *e) { + unsigned percent; + assert(e); + + if (e->written_uncompressed >= (uint64_t) e->st.st_size) + percent = 100; + else + percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / (uint64_t) e->st.st_size); + + if (percent == e->last_percent) + return; + + if (!ratelimit_below(&e->progress_ratelimit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Exported %u%%.", percent); + + e->last_percent = percent; +} + +static int raw_export_process(RawExport *e) { + ssize_t l; + int r; + + assert(e); + + if (!e->tried_reflink && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + /* If we shall take an uncompressed snapshot we can + * reflink source to destination directly. Let's see + * if this works. */ + + r = btrfs_reflink(e->input_fd, e->output_fd); + if (r >= 0) { + r = 0; + goto finish; + } + + e->tried_reflink = true; + } + + if (!e->tried_sendfile && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + l = sendfile(e->output_fd, e->input_fd, NULL, COPY_BUFFER_SIZE); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + e->tried_sendfile = true; + } else if (l == 0) { + r = 0; + goto finish; + } else { + e->written_uncompressed += l; + e->written_compressed += l; + + raw_export_report_progress(e); + + return 0; + } + } + + while (e->buffer_size <= 0) { + uint8_t input[COPY_BUFFER_SIZE]; + + if (e->eof) { + r = 0; + goto finish; + } + + l = read(e->input_fd, input, sizeof(input)); + if (l < 0) { + r = log_error_errno(errno, "Failed to read raw file: %m"); + goto finish; + } + + if (l == 0) { + e->eof = true; + r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } else { + e->written_uncompressed += l; + r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } + if (r < 0) { + r = log_error_errno(r, "Failed to encode: %m"); + goto finish; + } + } + + l = write(e->output_fd, e->buffer, e->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to write output file: %m"); + goto finish; + } + + assert((size_t) l <= e->buffer_size); + memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l); + e->buffer_size -= l; + e->written_compressed += l; + + raw_export_report_progress(e); + + return 0; + +finish: + if (r >= 0) { + (void) copy_times(e->input_fd, e->output_fd, COPY_CRTIME); + (void) copy_xattr(e->input_fd, e->output_fd, 0); + } + + if (e->on_finished) + e->on_finished(e, r, e->userdata); + else + sd_event_exit(e->event, r); + + return 0; +} + +static int raw_export_on_output(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + RawExport *i = userdata; + + return raw_export_process(i); +} + +static int raw_export_on_defer(sd_event_source *s, void *userdata) { + RawExport *i = userdata; + + return raw_export_process(i); +} + +static int reflink_snapshot(int fd, const char *path) { + int new_fd, r; + + new_fd = open_parent(path, O_TMPFILE|O_CLOEXEC|O_RDWR, 0600); + if (new_fd < 0) { + _cleanup_free_ char *t = NULL; + + r = tempfn_random(path, NULL, &t); + if (r < 0) + return r; + + new_fd = open(t, O_CLOEXEC|O_CREAT|O_NOCTTY|O_RDWR, 0600); + if (new_fd < 0) + return -errno; + + (void) unlink(t); + } + + r = btrfs_reflink(fd, new_fd); + if (r < 0) { + safe_close(new_fd); + return r; + } + + return new_fd; +} + +int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType compress) { + _cleanup_close_ int sfd = -1, tfd = -1; + int r; + + assert(e); + assert(path); + assert(fd >= 0); + assert(compress < _IMPORT_COMPRESS_TYPE_MAX); + assert(compress != IMPORT_COMPRESS_UNKNOWN); + + if (e->output_fd >= 0) + return -EBUSY; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&e->path, path); + if (r < 0) + return r; + + sfd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (sfd < 0) + return -errno; + + if (fstat(sfd, &e->st) < 0) + return -errno; + r = stat_verify_regular(&e->st); + if (r < 0) + return r; + + /* Try to take a reflink snapshot of the file, if we can t make the export atomic */ + tfd = reflink_snapshot(sfd, path); + if (tfd >= 0) + e->input_fd = TAKE_FD(tfd); + else + e->input_fd = TAKE_FD(sfd); + + r = import_compress_init(&e->compress, compress); + if (r < 0) + return r; + + r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, raw_export_on_output, e); + if (r == -EPERM) { + r = sd_event_add_defer(e->event, &e->output_event_source, raw_export_on_defer, e); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + e->output_fd = fd; + return r; +} diff --git a/src/import/export-raw.h b/src/import/export-raw.h new file mode 100644 index 0000000..27009e4 --- /dev/null +++ b/src/import/export-raw.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "sd-event.h" + +#include "import-compress.h" +#include "macro.h" + +typedef struct RawExport RawExport; + +typedef void (*RawExportFinished)(RawExport *export, int error, void *userdata); + +int raw_export_new(RawExport **export, sd_event *event, RawExportFinished on_finished, void *userdata); +RawExport* raw_export_unref(RawExport *export); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawExport*, raw_export_unref); + +int raw_export_start(RawExport *export, const char *path, int fd, ImportCompressType compress); diff --git a/src/import/export-tar.c b/src/import/export-tar.c new file mode 100644 index 0000000..78f0a7c --- /dev/null +++ b/src/import/export-tar.c @@ -0,0 +1,327 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "export-tar.h" +#include "fd-util.h" +#include "import-common.h" +#include "process-util.h" +#include "ratelimit.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +#define COPY_BUFFER_SIZE (16*1024) + +struct TarExport { + sd_event *event; + + TarExportFinished on_finished; + void *userdata; + + char *path; + char *temp_path; + + int output_fd; + int tar_fd; + + ImportCompress compress; + + sd_event_source *output_event_source; + + void *buffer; + size_t buffer_size; + size_t buffer_allocated; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + pid_t tar_pid; + + struct stat st; + uint64_t quota_referenced; + + unsigned last_percent; + RateLimit progress_ratelimit; + + bool eof; + bool tried_splice; +}; + +TarExport *tar_export_unref(TarExport *e) { + if (!e) + return NULL; + + sd_event_source_unref(e->output_event_source); + + if (e->tar_pid > 1) + sigkill_wait(e->tar_pid); + + if (e->temp_path) { + (void) btrfs_subvol_remove(e->temp_path, BTRFS_REMOVE_QUOTA); + free(e->temp_path); + } + + import_compress_free(&e->compress); + + sd_event_unref(e->event); + + safe_close(e->tar_fd); + + free(e->buffer); + free(e->path); + return mfree(e); +} + +int tar_export_new( + TarExport **ret, + sd_event *event, + TarExportFinished on_finished, + void *userdata) { + + _cleanup_(tar_export_unrefp) TarExport *e = NULL; + int r; + + assert(ret); + + e = new(TarExport, 1); + if (!e) + return -ENOMEM; + + *e = (TarExport) { + .output_fd = -1, + .tar_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .quota_referenced = UINT64_MAX, + .last_percent = UINT_MAX, + .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 }, + }; + + if (event) + e->event = sd_event_ref(event); + else { + r = sd_event_default(&e->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(e); + + return 0; +} + +static void tar_export_report_progress(TarExport *e) { + unsigned percent; + assert(e); + + /* Do we have any quota info? If not, we don't know anything about the progress */ + if (e->quota_referenced == UINT64_MAX) + return; + + if (e->written_uncompressed >= e->quota_referenced) + percent = 100; + else + percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / e->quota_referenced); + + if (percent == e->last_percent) + return; + + if (!ratelimit_below(&e->progress_ratelimit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Exported %u%%.", percent); + + e->last_percent = percent; +} + +static int tar_export_finish(TarExport *e) { + int r; + + assert(e); + assert(e->tar_fd >= 0); + + if (e->tar_pid > 0) { + r = wait_for_terminate_and_check("tar", TAKE_PID(e->tar_pid), WAIT_LOG); + if (r < 0) + return r; + if (r != EXIT_SUCCESS) + return -EPROTO; + } + + e->tar_fd = safe_close(e->tar_fd); + + return 0; +} + +static int tar_export_process(TarExport *e) { + ssize_t l; + int r; + + assert(e); + + if (!e->tried_splice && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + l = splice(e->tar_fd, NULL, e->output_fd, NULL, COPY_BUFFER_SIZE, 0); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + e->tried_splice = true; + } else if (l == 0) { + r = tar_export_finish(e); + goto finish; + } else { + e->written_uncompressed += l; + e->written_compressed += l; + + tar_export_report_progress(e); + + return 0; + } + } + + while (e->buffer_size <= 0) { + uint8_t input[COPY_BUFFER_SIZE]; + + if (e->eof) { + r = tar_export_finish(e); + goto finish; + } + + l = read(e->tar_fd, input, sizeof(input)); + if (l < 0) { + r = log_error_errno(errno, "Failed to read tar file: %m"); + goto finish; + } + + if (l == 0) { + e->eof = true; + r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } else { + e->written_uncompressed += l; + r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } + if (r < 0) { + r = log_error_errno(r, "Failed to encode: %m"); + goto finish; + } + } + + l = write(e->output_fd, e->buffer, e->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to write output file: %m"); + goto finish; + } + + assert((size_t) l <= e->buffer_size); + memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l); + e->buffer_size -= l; + e->written_compressed += l; + + tar_export_report_progress(e); + + return 0; + +finish: + if (e->on_finished) + e->on_finished(e, r, e->userdata); + else + sd_event_exit(e->event, r); + + return 0; +} + +static int tar_export_on_output(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + TarExport *i = userdata; + + return tar_export_process(i); +} + +static int tar_export_on_defer(sd_event_source *s, void *userdata) { + TarExport *i = userdata; + + return tar_export_process(i); +} + +int tar_export_start(TarExport *e, const char *path, int fd, ImportCompressType compress) { + _cleanup_close_ int sfd = -1; + int r; + + assert(e); + assert(path); + assert(fd >= 0); + assert(compress < _IMPORT_COMPRESS_TYPE_MAX); + assert(compress != IMPORT_COMPRESS_UNKNOWN); + + if (e->output_fd >= 0) + return -EBUSY; + + sfd = open(path, O_DIRECTORY|O_RDONLY|O_NOCTTY|O_CLOEXEC); + if (sfd < 0) + return -errno; + + if (fstat(sfd, &e->st) < 0) + return -errno; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&e->path, path); + if (r < 0) + return r; + + e->quota_referenced = UINT64_MAX; + + if (btrfs_might_be_subvol(&e->st)) { + BtrfsQuotaInfo q; + + r = btrfs_subvol_get_subtree_quota_fd(sfd, 0, &q); + if (r >= 0) + e->quota_referenced = q.referenced; + + e->temp_path = mfree(e->temp_path); + + r = tempfn_random(path, NULL, &e->temp_path); + if (r < 0) + return r; + + /* Let's try to make a snapshot, if we can, so that the export is atomic */ + r = btrfs_subvol_snapshot_fd(sfd, e->temp_path, BTRFS_SNAPSHOT_READ_ONLY|BTRFS_SNAPSHOT_RECURSIVE); + if (r < 0) { + log_debug_errno(r, "Couldn't create snapshot %s of %s, not exporting atomically: %m", e->temp_path, path); + e->temp_path = mfree(e->temp_path); + } + } + + r = import_compress_init(&e->compress, compress); + if (r < 0) + return r; + + r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, tar_export_on_output, e); + if (r == -EPERM) { + r = sd_event_add_defer(e->event, &e->output_event_source, tar_export_on_defer, e); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + e->tar_fd = import_fork_tar_c(e->temp_path ?: e->path, &e->tar_pid); + if (e->tar_fd < 0) { + e->output_event_source = sd_event_source_unref(e->output_event_source); + return e->tar_fd; + } + + e->output_fd = fd; + return r; +} diff --git a/src/import/export-tar.h b/src/import/export-tar.h new file mode 100644 index 0000000..3b55d12 --- /dev/null +++ b/src/import/export-tar.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "sd-event.h" + +#include "import-compress.h" +#include "macro.h" + +typedef struct TarExport TarExport; + +typedef void (*TarExportFinished)(TarExport *export, int error, void *userdata); + +int tar_export_new(TarExport **export, sd_event *event, TarExportFinished on_finished, void *userdata); +TarExport* tar_export_unref(TarExport *export); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarExport*, tar_export_unref); + +int tar_export_start(TarExport *export, const char *path, int fd, ImportCompressType compress); diff --git a/src/import/export.c b/src/import/export.c new file mode 100644 index 0000000..26533ba --- /dev/null +++ b/src/import/export.c @@ -0,0 +1,297 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <getopt.h> +#include <locale.h> + +#include "sd-event.h" +#include "sd-id128.h" + +#include "alloc-util.h" +#include "discover-image.h" +#include "export-raw.h" +#include "export-tar.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-util.h" +#include "main-func.h" +#include "signal-util.h" +#include "string-util.h" +#include "verbs.h" + +static ImportCompressType arg_compress = IMPORT_COMPRESS_UNKNOWN; + +static void determine_compression_from_filename(const char *p) { + + if (arg_compress != IMPORT_COMPRESS_UNKNOWN) + return; + + if (!p) { + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; + return; + } + + if (endswith(p, ".xz")) + arg_compress = IMPORT_COMPRESS_XZ; + else if (endswith(p, ".gz")) + arg_compress = IMPORT_COMPRESS_GZIP; + else if (endswith(p, ".bz2")) + arg_compress = IMPORT_COMPRESS_BZIP2; + else + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; +} + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +static void on_tar_finished(TarExport *export, int error, void *userdata) { + sd_event *event = userdata; + assert(export); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int export_tar(int argc, char *argv[], void *userdata) { + _cleanup_(tar_export_unrefp) TarExport *export = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(image_unrefp) Image *image = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (hostname_is_valid(argv[1], 0)) { + r = image_find(IMAGE_MACHINE, argv[1], NULL, &image); + if (r == -ENOENT) + return log_error_errno(r, "Machine image %s not found.", argv[1]); + if (r < 0) + return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + + local = image->path; + } else + local = argv[1]; + + if (argc >= 3) + path = argv[2]; + path = empty_or_dash_to_null(path); + + determine_compression_from_filename(path); + + if (path) { + open_fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open tar image for export: %m"); + + fd = open_fd; + + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDOUT_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = tar_export_new(&export, event, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate exporter: %m"); + + r = tar_export_start(export, local, fd, arg_compress); + if (r < 0) + return log_error_errno(r, "Failed to export image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawExport *export, int error, void *userdata) { + sd_event *event = userdata; + assert(export); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int export_raw(int argc, char *argv[], void *userdata) { + _cleanup_(raw_export_unrefp) RawExport *export = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(image_unrefp) Image *image = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (hostname_is_valid(argv[1], 0)) { + r = image_find(IMAGE_MACHINE, argv[1], NULL, &image); + if (r == -ENOENT) + return log_error_errno(r, "Machine image %s not found.", argv[1]); + if (r < 0) + return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + + local = image->path; + } else + local = argv[1]; + + if (argc >= 3) + path = argv[2]; + path = empty_or_dash_to_null(path); + + determine_compression_from_filename(path); + + if (path) { + open_fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open raw image for export: %m"); + + fd = open_fd; + + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDOUT_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = raw_export_new(&export, event, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate exporter: %m"); + + r = raw_export_start(export, local, fd, arg_compress); + if (r < 0) + return log_error_errno(r, "Failed to export image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%s [OPTIONS...] {COMMAND} ...\n\n" + "Export container or virtual machine images.\n\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --format=FORMAT Select format\n\n" + "Commands:\n" + " tar NAME [FILE] Export a TAR image\n" + " raw NAME [FILE] Export a RAW image\n", + program_invocation_short_name); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORMAT, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "format", required_argument, NULL, ARG_FORMAT }, + {} + }; + + int c; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORMAT: + if (streq(optarg, "uncompressed")) + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; + else if (streq(optarg, "xz")) + arg_compress = IMPORT_COMPRESS_XZ; + else if (streq(optarg, "gzip")) + arg_compress = IMPORT_COMPRESS_GZIP; + else if (streq(optarg, "bzip2")) + arg_compress = IMPORT_COMPRESS_BZIP2; + else + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Unknown format: %s", optarg); + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached(); + } + + return 1; +} + +static int export_main(int argc, char *argv[]) { + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, export_tar }, + { "raw", 2, 3, 0, export_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +static int run(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + return r; + + (void) ignore_signals(SIGPIPE); + + return export_main(argc, argv); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/import-common.c b/src/import/import-common.c new file mode 100644 index 0000000..9e64e53 --- /dev/null +++ b/src/import/import-common.c @@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <sched.h> +#include <sys/prctl.h> +#include <sys/stat.h> +#include <unistd.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "capability-util.h" +#include "chattr-util.h" +#include "dirent-util.h" +#include "fd-util.h" +#include "fileio.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "os-util.h" +#include "process-util.h" +#include "selinux-util.h" +#include "signal-util.h" +#include "stat-util.h" +#include "tmpfile-util.h" +#include "util.h" + +int import_fork_tar_x(const char *path, pid_t *ret) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + bool use_selinux; + pid_t pid; + int r; + + assert(path); + assert(ret); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return log_error_errno(errno, "Failed to create pipe for tar: %m"); + + use_selinux = mac_selinux_use(); + + r = safe_fork("(tar)", FORK_RESET_SIGNALS|FORK_DEATHSIG|FORK_LOG, &pid); + if (r < 0) + return r; + if (r == 0) { + const char *cmdline[] = { + "tar", + "--ignore-zeros", + "--numeric-owner", + "-C", path, + "-pxf", + "-", + "--xattrs", + "--xattrs-include=*", + use_selinux ? "--selinux" : "--no-selinux", + NULL + }; + + uint64_t retain = + (1ULL << CAP_CHOWN) | + (1ULL << CAP_FOWNER) | + (1ULL << CAP_FSETID) | + (1ULL << CAP_MKNOD) | + (1ULL << CAP_SETFCAP) | + (1ULL << CAP_DAC_OVERRIDE); + + /* Child */ + + pipefd[1] = safe_close(pipefd[1]); + + r = rearrange_stdio(TAKE_FD(pipefd[0]), -1, STDERR_FILENO); + if (r < 0) { + log_error_errno(r, "Failed to rearrange stdin/stdout: %m"); + _exit(EXIT_FAILURE); + } + + if (unshare(CLONE_NEWNET) < 0) + log_warning_errno(errno, "Failed to lock tar into network namespace, ignoring: %m"); + + r = capability_bounding_set_drop(retain, true); + if (r < 0) + log_warning_errno(r, "Failed to drop capabilities, ignoring: %m"); + + /* Try "gtar" before "tar". We only test things upstream with GNU tar. Some distros appear to + * install a different implementation as "tar" (in particular some that do not support the + * same command line switches), but then provide "gtar" as alias for the real thing, hence + * let's prefer that. (Yes, it's a bad idea they do that, given they don't provide equivalent + * command line support, but we are not here to argue, let's just expose the same + * behaviour/implementation everywhere.) */ + execvp("gtar", (char* const*) cmdline); + execvp("tar", (char* const*) cmdline); + + log_error_errno(errno, "Failed to execute tar: %m"); + _exit(EXIT_FAILURE); + } + + *ret = pid; + + return TAKE_FD(pipefd[1]); +} + +int import_fork_tar_c(const char *path, pid_t *ret) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + bool use_selinux; + pid_t pid; + int r; + + assert(path); + assert(ret); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return log_error_errno(errno, "Failed to create pipe for tar: %m"); + + use_selinux = mac_selinux_use(); + + r = safe_fork("(tar)", FORK_RESET_SIGNALS|FORK_DEATHSIG|FORK_LOG, &pid); + if (r < 0) + return r; + if (r == 0) { + const char *cmdline[] = { + "tar", + "-C", path, + "-c", + "--xattrs", + "--xattrs-include=*", + use_selinux ? "--selinux" : "--no-selinux", + ".", + NULL + }; + + uint64_t retain = (1ULL << CAP_DAC_OVERRIDE); + + /* Child */ + + pipefd[0] = safe_close(pipefd[0]); + + r = rearrange_stdio(-1, TAKE_FD(pipefd[1]), STDERR_FILENO); + if (r < 0) { + log_error_errno(r, "Failed to rearrange stdin/stdout: %m"); + _exit(EXIT_FAILURE); + } + + if (unshare(CLONE_NEWNET) < 0) + log_error_errno(errno, "Failed to lock tar into network namespace, ignoring: %m"); + + r = capability_bounding_set_drop(retain, true); + if (r < 0) + log_error_errno(r, "Failed to drop capabilities, ignoring: %m"); + + execvp("gtar", (char* const*) cmdline); + execvp("tar", (char* const*) cmdline); + + log_error_errno(errno, "Failed to execute tar: %m"); + _exit(EXIT_FAILURE); + } + + *ret = pid; + + return TAKE_FD(pipefd[0]); +} + +int import_mangle_os_tree(const char *path) { + _cleanup_free_ char *child = NULL, *t = NULL, *joined = NULL; + _cleanup_closedir_ DIR *d = NULL, *cd = NULL; + struct dirent *dent; + struct stat st; + int r; + + assert(path); + + /* Some tarballs contain a single top-level directory that contains the actual OS directory tree. Try to + * recognize this, and move the tree one level up. */ + + r = path_is_os_tree(path); + if (r < 0) + return log_error_errno(r, "Failed to determine whether '%s' is an OS tree: %m", path); + if (r > 0) { + log_debug("Directory tree '%s' is a valid OS tree.", path); + return 0; + } + + log_debug("Directory tree '%s' is not recognizable as OS tree, checking whether to rearrange it.", path); + + d = opendir(path); + if (!d) + return log_error_errno(r, "Failed to open directory '%s': %m", path); + + errno = 0; + dent = readdir_no_dot(d); + if (!dent) { + if (errno != 0) + return log_error_errno(errno, "Failed to iterate through directory '%s': %m", path); + + log_debug("Directory '%s' is empty, leaving it as it is.", path); + return 0; + } + + child = strdup(dent->d_name); + if (!child) + return log_oom(); + + errno = 0; + dent = readdir_no_dot(d); + if (dent) { + if (errno != 0) + return log_error_errno(errno, "Failed to iterate through directory '%s': %m", path); + + log_debug("Directory '%s' does not look like an OS tree, and has multiple children, leaving as it is.", path); + return 0; + } + + if (fstatat(dirfd(d), child, &st, AT_SYMLINK_NOFOLLOW) < 0) + return log_debug_errno(errno, "Failed to stat file '%s/%s': %m", path, child); + r = stat_verify_directory(&st); + if (r < 0) { + log_debug_errno(r, "Child '%s' of directory '%s' is not a directory, leaving things as they are.", child, path); + return 0; + } + + joined = path_join(path, child); + if (!joined) + return log_oom(); + r = path_is_os_tree(joined); + if (r == -ENOTDIR) { + log_debug("Directory '%s' does not look like an OS tree, and contains a single regular file only, leaving as it is.", path); + return 0; + } + if (r < 0) + return log_error_errno(r, "Failed to determine whether '%s' is an OS tree: %m", joined); + if (r == 0) { + log_debug("Neither '%s' nor '%s' is a valid OS tree, leaving them as they are.", path, joined); + return 0; + } + + /* Nice, we have checked now: + * + * 1. The top-level directory does not qualify as OS tree + * 1. The top-level directory only contains one item + * 2. That item is a directory + * 3. And that directory qualifies as OS tree + * + * Let's now rearrange things, moving everything in the inner directory one level up */ + + cd = xopendirat(dirfd(d), child, O_NOFOLLOW); + if (!cd) + return log_error_errno(errno, "Can't open directory '%s': %m", joined); + + log_info("Rearranging '%s', moving OS tree one directory up.", joined); + + /* Let's rename the child to an unguessable name so that we can be sure all files contained in it can be + * safely moved up and won't collide with the name. */ + r = tempfn_random(child, NULL, &t); + if (r < 0) + return log_oom(); + r = rename_noreplace(dirfd(d), child, dirfd(d), t); + if (r < 0) + return log_error_errno(r, "Unable to rename '%s' to '%s/%s': %m", joined, path, t); + + FOREACH_DIRENT_ALL(de, cd, return log_error_errno(errno, "Failed to iterate through directory '%s': %m", joined)) { + if (dot_or_dot_dot(de->d_name)) + continue; + + r = rename_noreplace(dirfd(cd), de->d_name, dirfd(d), de->d_name); + if (r < 0) + return log_error_errno(r, "Unable to move '%s/%s/%s' to '%s/%s': %m", path, t, de->d_name, path, de->d_name); + } + + if (unlinkat(dirfd(d), t, AT_REMOVEDIR) < 0) + return log_error_errno(errno, "Failed to remove temporary directory '%s/%s': %m", path, t); + + r = futimens(dirfd(d), (struct timespec[2]) { st.st_atim, st.st_mtim }); + if (r < 0) + log_debug_errno(r, "Failed to adjust top-level timestamps '%s', ignoring: %m", path); + + r = fchmod_and_chown(dirfd(d), st.st_mode, st.st_uid, st.st_gid); + if (r < 0) + return log_error_errno(r, "Failed to adjust top-level directory mode/ownership '%s': %m", path); + + log_info("Successfully rearranged OS tree."); + + return 0; +} + +bool import_validate_local(const char *name, ImportFlags flags) { + + /* By default we insist on a valid hostname for naming images. But optionally we relax that, in which + * case it can be any path name */ + + if (FLAGS_SET(flags, IMPORT_DIRECT)) + return path_is_valid(name); + + return hostname_is_valid(name, 0); +} + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +int import_allocate_event_with_signals(sd_event **ret) { + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + int r; + + assert(ret); + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + *ret = TAKE_PTR(event); + return 0; +} diff --git a/src/import/import-common.h b/src/import/import-common.h new file mode 100644 index 0000000..97fc16d --- /dev/null +++ b/src/import/import-common.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include <sys/types.h> + +#include "sd-event.h" + +typedef enum ImportFlags { + IMPORT_FORCE = 1 << 0, /* replace existing image */ + IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */ + IMPORT_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */ + IMPORT_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ + IMPORT_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */ + IMPORT_DIRECT = 1 << 5, /* import without rename games */ + IMPORT_SYNC = 1 << 6, /* fsync() right before we are done */ + + IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC, + IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC, +} ImportFlags; + +int import_fork_tar_c(const char *path, pid_t *ret); +int import_fork_tar_x(const char *path, pid_t *ret); + +int import_mangle_os_tree(const char *path); + +bool import_validate_local(const char *name, ImportFlags flags); + +int import_allocate_event_with_signals(sd_event **ret); diff --git a/src/import/import-compress.c b/src/import/import-compress.c new file mode 100644 index 0000000..03f5efa --- /dev/null +++ b/src/import/import-compress.c @@ -0,0 +1,479 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "import-compress.h" +#include "string-table.h" +#include "util.h" + +void import_compress_free(ImportCompress *c) { + assert(c); + + if (c->type == IMPORT_COMPRESS_XZ) + lzma_end(&c->xz); + else if (c->type == IMPORT_COMPRESS_GZIP) { + if (c->encoding) + deflateEnd(&c->gzip); + else + inflateEnd(&c->gzip); +#if HAVE_BZIP2 + } else if (c->type == IMPORT_COMPRESS_BZIP2) { + if (c->encoding) + BZ2_bzCompressEnd(&c->bzip2); + else + BZ2_bzDecompressEnd(&c->bzip2); +#endif + } + + c->type = IMPORT_COMPRESS_UNKNOWN; +} + +int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) { + static const uint8_t xz_signature[] = { + 0xfd, '7', 'z', 'X', 'Z', 0x00 + }; + static const uint8_t gzip_signature[] = { + 0x1f, 0x8b + }; + static const uint8_t bzip2_signature[] = { + 'B', 'Z', 'h' + }; + + int r; + + assert(c); + + if (c->type != IMPORT_COMPRESS_UNKNOWN) + return 1; + + if (size < MAX3(sizeof(xz_signature), + sizeof(gzip_signature), + sizeof(bzip2_signature))) + return 0; + + assert(data); + + if (memcmp(data, xz_signature, sizeof(xz_signature)) == 0) { + lzma_ret xzr; + + xzr = lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED); + if (xzr != LZMA_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_XZ; + + } else if (memcmp(data, gzip_signature, sizeof(gzip_signature)) == 0) { + r = inflateInit2(&c->gzip, 15+16); + if (r != Z_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_GZIP; + +#if HAVE_BZIP2 + } else if (memcmp(data, bzip2_signature, sizeof(bzip2_signature)) == 0) { + r = BZ2_bzDecompressInit(&c->bzip2, 0, 0); + if (r != BZ_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_BZIP2; +#endif + } else + c->type = IMPORT_COMPRESS_UNCOMPRESSED; + + c->encoding = false; + + return 1; +} + +void import_uncompress_force_off(ImportCompress *c) { + assert(c); + + c->type = IMPORT_COMPRESS_UNCOMPRESSED; + c->encoding = false; +} + +int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata) { + int r; + + assert(c); + assert(callback); + + r = import_uncompress_detect(c, data, size); + if (r <= 0) + return r; + + if (c->encoding) + return -EINVAL; + + if (size <= 0) + return 1; + + assert(data); + + switch (c->type) { + + case IMPORT_COMPRESS_UNCOMPRESSED: + r = callback(data, size, userdata); + if (r < 0) + return r; + + break; + + case IMPORT_COMPRESS_XZ: + c->xz.next_in = data; + c->xz.avail_in = size; + + while (c->xz.avail_in > 0) { + uint8_t buffer[16 * 1024]; + lzma_ret lzr; + + c->xz.next_out = buffer; + c->xz.avail_out = sizeof(buffer); + + lzr = lzma_code(&c->xz, LZMA_RUN); + if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) + return -EIO; + + if (c->xz.avail_out < sizeof(buffer)) { + r = callback(buffer, sizeof(buffer) - c->xz.avail_out, userdata); + if (r < 0) + return r; + } + } + + break; + + case IMPORT_COMPRESS_GZIP: + c->gzip.next_in = (void*) data; + c->gzip.avail_in = size; + + while (c->gzip.avail_in > 0) { + uint8_t buffer[16 * 1024]; + + c->gzip.next_out = buffer; + c->gzip.avail_out = sizeof(buffer); + + r = inflate(&c->gzip, Z_NO_FLUSH); + if (!IN_SET(r, Z_OK, Z_STREAM_END)) + return -EIO; + + if (c->gzip.avail_out < sizeof(buffer)) { + r = callback(buffer, sizeof(buffer) - c->gzip.avail_out, userdata); + if (r < 0) + return r; + } + } + + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + c->bzip2.next_in = (void*) data; + c->bzip2.avail_in = size; + + while (c->bzip2.avail_in > 0) { + uint8_t buffer[16 * 1024]; + + c->bzip2.next_out = (char*) buffer; + c->bzip2.avail_out = sizeof(buffer); + + r = BZ2_bzDecompress(&c->bzip2); + if (!IN_SET(r, BZ_OK, BZ_STREAM_END)) + return -EIO; + + if (c->bzip2.avail_out < sizeof(buffer)) { + r = callback(buffer, sizeof(buffer) - c->bzip2.avail_out, userdata); + if (r < 0) + return r; + } + } + + break; +#endif + + default: + assert_not_reached(); + } + + return 1; +} + +int import_compress_init(ImportCompress *c, ImportCompressType t) { + int r; + + assert(c); + + switch (t) { + + case IMPORT_COMPRESS_XZ: { + lzma_ret xzr; + + xzr = lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); + if (xzr != LZMA_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_XZ; + break; + } + + case IMPORT_COMPRESS_GZIP: + r = deflateInit2(&c->gzip, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY); + if (r != Z_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_GZIP; + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + r = BZ2_bzCompressInit(&c->bzip2, 9, 0, 0); + if (r != BZ_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_BZIP2; + break; +#endif + + case IMPORT_COMPRESS_UNCOMPRESSED: + c->type = IMPORT_COMPRESS_UNCOMPRESSED; + break; + + default: + return -EOPNOTSUPP; + } + + c->encoding = true; + return 0; +} + +static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + size_t l; + void *p; + + if (*buffer_allocated > *buffer_size) + return 0; + + l = MAX(16*1024U, (*buffer_size * 2)); + p = realloc(*buffer, l); + if (!p) + return -ENOMEM; + + *buffer = p; + *buffer_allocated = l; + + return 1; +} + +int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + int r; + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + if (size <= 0) + return 0; + + assert(data); + + *buffer_size = 0; + + switch (c->type) { + + case IMPORT_COMPRESS_XZ: + + c->xz.next_in = data; + c->xz.avail_in = size; + + while (c->xz.avail_in > 0) { + lzma_ret lzr; + + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = lzma_code(&c->xz, LZMA_RUN); + if (lzr != LZMA_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } + + break; + + case IMPORT_COMPRESS_GZIP: + + c->gzip.next_in = (void*) data; + c->gzip.avail_in = size; + + while (c->gzip.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = *buffer_allocated - *buffer_size; + + r = deflate(&c->gzip, Z_NO_FLUSH); + if (r != Z_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; + } + + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + + c->bzip2.next_in = (void*) data; + c->bzip2.avail_in = size; + + while (c->bzip2.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = *buffer_allocated - *buffer_size; + + r = BZ2_bzCompress(&c->bzip2, BZ_RUN); + if (r != BZ_RUN_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; + } + + break; +#endif + + case IMPORT_COMPRESS_UNCOMPRESSED: + + if (*buffer_allocated < size) { + void *p; + + p = realloc(*buffer, size); + if (!p) + return -ENOMEM; + + *buffer = p; + *buffer_allocated = size; + } + + memcpy(*buffer, data, size); + *buffer_size = size; + break; + + default: + return -EOPNOTSUPP; + } + + return 0; +} + +int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + int r; + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + *buffer_size = 0; + + switch (c->type) { + + case IMPORT_COMPRESS_XZ: { + lzma_ret lzr; + + c->xz.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = lzma_code(&c->xz, LZMA_FINISH); + if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } while (lzr != LZMA_STREAM_END); + + break; + } + + case IMPORT_COMPRESS_GZIP: + c->gzip.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = *buffer_allocated - *buffer_size; + + r = deflate(&c->gzip, Z_FINISH); + if (!IN_SET(r, Z_OK, Z_STREAM_END)) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; + } while (r != Z_STREAM_END); + + break; + +#if HAVE_BZIP2 + case IMPORT_COMPRESS_BZIP2: + c->bzip2.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = *buffer_allocated - *buffer_size; + + r = BZ2_bzCompress(&c->bzip2, BZ_FINISH); + if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END)) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; + } while (r != BZ_STREAM_END); + + break; +#endif + + case IMPORT_COMPRESS_UNCOMPRESSED: + break; + + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static const char* const import_compress_type_table[_IMPORT_COMPRESS_TYPE_MAX] = { + [IMPORT_COMPRESS_UNKNOWN] = "unknown", + [IMPORT_COMPRESS_UNCOMPRESSED] = "uncompressed", + [IMPORT_COMPRESS_XZ] = "xz", + [IMPORT_COMPRESS_GZIP] = "gzip", +#if HAVE_BZIP2 + [IMPORT_COMPRESS_BZIP2] = "bzip2", +#endif +}; + +DEFINE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType); diff --git a/src/import/import-compress.h b/src/import/import-compress.h new file mode 100644 index 0000000..0a42103 --- /dev/null +++ b/src/import/import-compress.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#if HAVE_BZIP2 +#include <bzlib.h> +#endif +#include <lzma.h> +#include <sys/types.h> +#include <zlib.h> + +#include "macro.h" + +typedef enum ImportCompressType { + IMPORT_COMPRESS_UNKNOWN, + IMPORT_COMPRESS_UNCOMPRESSED, + IMPORT_COMPRESS_XZ, + IMPORT_COMPRESS_GZIP, + IMPORT_COMPRESS_BZIP2, + _IMPORT_COMPRESS_TYPE_MAX, + _IMPORT_COMPRESS_TYPE_INVALID = -EINVAL, +} ImportCompressType; + +typedef struct ImportCompress { + ImportCompressType type; + bool encoding; + union { + lzma_stream xz; + z_stream gzip; +#if HAVE_BZIP2 + bz_stream bzip2; +#endif + }; +} ImportCompress; + +typedef int (*ImportCompressCallback)(const void *data, size_t size, void *userdata); + +void import_compress_free(ImportCompress *c); + +int import_uncompress_detect(ImportCompress *c, const void *data, size_t size); +void import_uncompress_force_off(ImportCompress *c); +int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata); + +int import_compress_init(ImportCompress *c, ImportCompressType t); +int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated); +int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated); + +const char* import_compress_type_to_string(ImportCompressType t) _const_; +ImportCompressType import_compress_type_from_string(const char *s) _pure_; diff --git a/src/import/import-fs.c b/src/import/import-fs.c new file mode 100644 index 0000000..1c5581f --- /dev/null +++ b/src/import/import-fs.c @@ -0,0 +1,391 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <getopt.h> +#include <locale.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "discover-image.h" +#include "fd-util.h" +#include "format-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-util.h" +#include "install-file.h" +#include "main-func.h" +#include "mkdir-label.h" +#include "parse-argument.h" +#include "ratelimit.h" +#include "rm-rf.h" +#include "signal-util.h" +#include "string-util.h" +#include "terminal-util.h" +#include "tmpfile-util.h" +#include "verbs.h" + +static bool arg_force = false; +static bool arg_read_only = false; +static bool arg_btrfs_subvol = true; +static bool arg_btrfs_quota = true; +static bool arg_sync = true; +static bool arg_direct = false; +static const char *arg_image_root = "/var/lib/machines"; + +typedef struct ProgressInfo { + RateLimit limit; + char *path; + uint64_t size; + bool started; + bool logged_incomplete; +} ProgressInfo; + +static void progress_info_free(ProgressInfo *p) { + free(p->path); +} + +static void progress_show(ProgressInfo *p) { + assert(p); + + /* Show progress only every now and then. */ + if (!ratelimit_below(&p->limit)) + return; + + /* Suppress the first message, start with the second one */ + if (!p->started) { + p->started = true; + return; + } + + /* Mention the list is incomplete before showing first output. */ + if (!p->logged_incomplete) { + log_notice("(Note: file list shown below is incomplete, and is intended as sporadic progress report only.)"); + p->logged_incomplete = true; + } + + if (p->size == 0) + log_info("Copying tree, currently at '%s'...", p->path); + else + log_info("Copying tree, currently at '%s' (@%s)...", p->path, FORMAT_BYTES(p->size)); +} + +static int progress_path(const char *path, const struct stat *st, void *userdata) { + ProgressInfo *p = ASSERT_PTR(userdata); + int r; + + r = free_and_strdup(&p->path, path); + if (r < 0) + return r; + + p->size = 0; + + progress_show(p); + return 0; +} + +static int progress_bytes(uint64_t nbytes, void *userdata) { + ProgressInfo *p = ASSERT_PTR(userdata); + + assert(p->size != UINT64_MAX); + + p->size += nbytes; + + progress_show(p); + return 0; +} + +static int import_fs(int argc, char *argv[], void *userdata) { + _cleanup_(rm_rf_subvolume_and_freep) char *temp_path = NULL; + _cleanup_(progress_info_free) ProgressInfo progress = {}; + _cleanup_free_ char *l = NULL, *final_path = NULL; + const char *path = NULL, *local = NULL, *dest = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (argc >= 2) + path = empty_or_dash_to_null(argv[1]); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else if (path) { + r = path_extract_filename(path, &l); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from path '%s': %m", path); + + local = l; + } + + if (arg_direct) { + if (!local) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified."); + + if (path_is_absolute(local)) + final_path = strdup(local); + else + final_path = path_join(arg_image_root, local); + if (!final_path) + return log_oom(); + + if (!path_is_valid(final_path)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local path name '%s' is not valid.", final_path); + } else { + if (local) { + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local image name '%s' is not valid.", local); + } else + local = "imported"; + + final_path = path_join(arg_image_root, local); + if (!final_path) + return log_oom(); + + if (!arg_force) { + r = image_find(IMAGE_MACHINE, local, NULL, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else + return log_error_errno(SYNTHETIC_ERRNO(EEXIST), + "Image '%s' already exists.", local); + } + } + + if (path) { + open_fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open directory to import: %m"); + + fd = open_fd; + + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDIN_FILENO; + + (void) fd_get_path(fd, &pretty); + log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); + } + + if (!arg_sync) + log_info("File system synchronization on completion is off."); + + if (arg_direct) { + if (arg_force) + (void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + dest = final_path; + } else { + r = tempfn_random(final_path, NULL, &temp_path); + if (r < 0) + return log_oom(); + + dest = temp_path; + } + + (void) mkdir_parents_label(dest, 0700); + + progress.limit = (RateLimit) { 200*USEC_PER_MSEC, 1 }; + + { + BLOCK_SIGNALS(SIGINT, SIGTERM); + + if (arg_btrfs_subvol) + r = btrfs_subvol_snapshot_fd_full( + fd, + dest, + BTRFS_SNAPSHOT_FALLBACK_COPY| + BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| + BTRFS_SNAPSHOT_RECURSIVE| + BTRFS_SNAPSHOT_SIGINT| + BTRFS_SNAPSHOT_SIGTERM, + progress_path, + progress_bytes, + &progress); + else + r = copy_directory_fd_full( + fd, + dest, + COPY_REFLINK| + COPY_SAME_MOUNT| + COPY_HARDLINKS| + COPY_SIGINT| + COPY_SIGTERM| + (arg_direct ? COPY_MERGE_EMPTY : 0), + progress_path, + progress_bytes, + &progress); + if (r == -EINTR) /* SIGINT/SIGTERM hit */ + return log_error_errno(r, "Copy cancelled."); + if (r < 0) + return log_error_errno(r, "Failed to copy directory: %m"); + } + + r = import_mangle_os_tree(dest); + if (r < 0) + return r; + + if (arg_btrfs_quota) { + if (!arg_direct) + (void) import_assign_pool_quota_and_warn(arg_image_root); + (void) import_assign_pool_quota_and_warn(dest); + } + + r = install_file(AT_FDCWD, dest, + AT_FDCWD, arg_direct ? NULL : final_path, /* pass NULL as target in case of direct + * mode since file is already in place */ + (arg_force ? INSTALL_REPLACE : 0) | + (arg_read_only ? INSTALL_READ_ONLY : 0) | + (arg_sync ? INSTALL_SYNCFS : 0)); + if (r < 0) + return log_error_errno(r, "Failed install directory as '%s': %m", final_path); + + temp_path = mfree(temp_path); + + log_info("Directory '%s successfully installed. Exiting.", final_path); + return 0; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%1$s [OPTIONS...] {COMMAND} ...\n" + "\n%4$sImport container images from a file system directories.%5$s\n" + "\n%2$sCommands:%3$s\n" + " run DIRECTORY [NAME] Import a directory\n" + "\n%2$sOptions:%3$s\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --force Force creation of image\n" + " --image-root=PATH Image root directory\n" + " --read-only Create a read-only image\n" + " --direct Import directly to specified directory\n" + " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n" + " instead of a directory\n" + " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n" + " subvolume\n" + " --sync=BOOL Controls whether to sync() before completing\n", + program_invocation_short_name, + ansi_underline(), + ansi_normal(), + ansi_highlight(), + ansi_normal()); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORCE, + ARG_IMAGE_ROOT, + ARG_READ_ONLY, + ARG_DIRECT, + ARG_BTRFS_SUBVOL, + ARG_BTRFS_QUOTA, + ARG_SYNC, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "force", no_argument, NULL, ARG_FORCE }, + { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "direct", no_argument, NULL, ARG_DIRECT }, + { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL }, + { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA }, + { "sync", required_argument, NULL, ARG_SYNC }, + {} + }; + + int c, r; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORCE: + arg_force = true; + break; + + case ARG_IMAGE_ROOT: + arg_image_root = optarg; + break; + + case ARG_READ_ONLY: + arg_read_only = true; + break; + + case ARG_DIRECT: + arg_direct = true; + break; + + case ARG_BTRFS_SUBVOL: + r = parse_boolean_argument("--btrfs-subvol=", optarg, &arg_btrfs_subvol); + if (r < 0) + return r; + + break; + + case ARG_BTRFS_QUOTA: + r = parse_boolean_argument("--btrfs-quota=", optarg, &arg_btrfs_quota); + if (r < 0) + return r; + + break; + + case ARG_SYNC: + r = parse_boolean_argument("--sync=", optarg, &arg_sync); + if (r < 0) + return r; + + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached(); + } + + return 1; +} + +static int import_fs_main(int argc, char *argv[]) { + + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "run", 2, 3, 0, import_fs }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +static int run(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + return r; + + return import_fs_main(argc, argv); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/import-pubring.gpg b/src/import/import-pubring.gpg Binary files differnew file mode 100644 index 0000000..be27776 --- /dev/null +++ b/src/import/import-pubring.gpg diff --git a/src/import/import-raw.c b/src/import/import-raw.c new file mode 100644 index 0000000..30f5b90 --- /dev/null +++ b/src/import/import-raw.c @@ -0,0 +1,530 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <linux/fs.h> + +#include "sd-daemon.h" +#include "sd-event.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "fd-util.h" +#include "format-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-compress.h" +#include "import-raw.h" +#include "install-file.h" +#include "io-util.h" +#include "machine-pool.h" +#include "mkdir-label.h" +#include "path-util.h" +#include "qcow2-util.h" +#include "ratelimit.h" +#include "rm-rf.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +struct RawImport { + sd_event *event; + + char *image_root; + + RawImportFinished on_finished; + void *userdata; + + char *local; + ImportFlags flags; + + char *temp_path; + char *final_path; + + int input_fd; + int output_fd; + + ImportCompress compress; + + sd_event_source *input_event_source; + + uint8_t buffer[16*1024]; + size_t buffer_size; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + struct stat input_stat; + struct stat output_stat; + + unsigned last_percent; + RateLimit progress_ratelimit; + + uint64_t offset; + uint64_t size_max; +}; + +RawImport* raw_import_unref(RawImport *i) { + if (!i) + return NULL; + + sd_event_source_unref(i->input_event_source); + + unlink_and_free(i->temp_path); + + import_compress_free(&i->compress); + + sd_event_unref(i->event); + + safe_close(i->output_fd); + + free(i->final_path); + free(i->image_root); + free(i->local); + return mfree(i); +} + +int raw_import_new( + RawImport **ret, + sd_event *event, + const char *image_root, + RawImportFinished on_finished, + void *userdata) { + + _cleanup_(raw_import_unrefp) RawImport *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + i = new(RawImport, 1); + if (!i) + return -ENOMEM; + + *i = (RawImport) { + .input_fd = -1, + .output_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .last_percent = UINT_MAX, + .image_root = TAKE_PTR(root), + .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 }, + .offset = UINT64_MAX, + .size_max = UINT64_MAX, + }; + + if (event) + i->event = sd_event_ref(event); + else { + r = sd_event_default(&i->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(i); + return 0; +} + +static void raw_import_report_progress(RawImport *i) { + unsigned percent; + assert(i); + + /* We have no size information, unless the source is a regular file */ + if (!S_ISREG(i->input_stat.st_mode)) + return; + + if (i->written_compressed >= (uint64_t) i->input_stat.st_size) + percent = 100; + else + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size); + + if (percent == i->last_percent) + return; + + if (!ratelimit_below(&i->progress_ratelimit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Imported %u%%.", percent); + + i->last_percent = percent; +} + +static int raw_import_maybe_convert_qcow2(RawImport *i) { + _cleanup_close_ int converted_fd = -1; + _cleanup_(unlink_and_freep) char *t = NULL; + _cleanup_free_ char *f = NULL; + int r; + + assert(i); + + /* Do QCOW2 conversion if enabled and not in direct mode */ + if ((i->flags & (IMPORT_CONVERT_QCOW2|IMPORT_DIRECT)) != IMPORT_CONVERT_QCOW2) + return 0; + + assert(i->final_path); + + r = qcow2_detect(i->output_fd); + if (r < 0) + return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); + if (r == 0) + return 0; + + /* This is a QCOW2 image, let's convert it */ + r = tempfn_random(i->final_path, NULL, &f); + if (r < 0) + return log_oom(); + + converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (converted_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", f); + + t = TAKE_PTR(f); + + (void) import_set_nocow_and_log(converted_fd, t); + + log_info("Unpacking QCOW2 file."); + + r = qcow2_convert(i->output_fd, converted_fd); + if (r < 0) + return log_error_errno(r, "Failed to convert qcow2 image: %m"); + + unlink_and_free(i->temp_path); + i->temp_path = TAKE_PTR(t); + close_and_replace(i->output_fd, converted_fd); + + return 1; +} + +static int raw_import_finish(RawImport *i) { + int r; + + assert(i); + assert(i->output_fd >= 0); + + /* Nothing of what is below applies to block devices */ + if (S_ISBLK(i->output_stat.st_mode)) { + + if (i->flags & IMPORT_SYNC) { + if (fsync(i->output_fd) < 0) + return log_error_errno(errno, "Failed to synchronize block device: %m"); + } + + return 0; + } + + assert(S_ISREG(i->output_stat.st_mode)); + + /* If an offset is specified we only are supposed to affect part of an existing output file or block + * device, thus don't manipulate file properties in that case */ + + if (i->offset == UINT64_MAX) { + /* In case this was a sparse file, make sure the file size is right */ + if (i->written_uncompressed > 0) { + if (ftruncate(i->output_fd, i->written_uncompressed) < 0) + return log_error_errno(errno, "Failed to truncate file: %m"); + } + + r = raw_import_maybe_convert_qcow2(i); + if (r < 0) + return r; + + if (S_ISREG(i->input_stat.st_mode)) { + (void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME); + (void) copy_xattr(i->input_fd, i->output_fd, 0); + } + } + + r = install_file(AT_FDCWD, i->temp_path ?: i->local, + AT_FDCWD, i->final_path, + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) + return log_error_errno(r, "Failed to move image into place: %m"); + + i->temp_path = mfree(i->temp_path); + + log_info("Wrote %s.", FORMAT_BYTES(i->written_uncompressed)); + + return 0; +} + +static int raw_import_open_disk(RawImport *i) { + int r; + + assert(i); + assert(i->local); + assert(!i->final_path); + assert(!i->temp_path); + assert(i->output_fd < 0); + + if (i->flags & IMPORT_DIRECT) { + (void) mkdir_parents_label(i->local, 0700); + + /* In direct mode we just open/create the local path and truncate it (like shell > + * redirection would do it) — except if an offset was passed, in which case we are supposed + * to operate on a section of the file only, in which case we apparently work on an some + * existing thing (i.e. are not the sole thing stored in the file), in which case we will + * neither truncate nor create. */ + + i->output_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664); + if (i->output_fd < 0) + return log_error_errno(errno, "Failed to open destination '%s': %m", i->local); + + if (i->offset == UINT64_MAX) + (void) import_set_nocow_and_log(i->output_fd, i->local); + } else { + i->final_path = strjoin(i->image_root, "/", i->local, ".raw"); + if (!i->final_path) + return log_oom(); + + r = tempfn_random(i->final_path, NULL, &i->temp_path); + if (r < 0) + return log_oom(); + + (void) mkdir_parents_label(i->temp_path, 0700); + + i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (i->output_fd < 0) + return log_error_errno(errno, "Failed to open destination '%s': %m", i->temp_path); + + (void) import_set_nocow_and_log(i->output_fd, i->temp_path); + } + + if (fstat(i->output_fd, &i->output_stat) < 0) + return log_error_errno(errno, "Failed to stat() output file: %m"); + + if (!S_ISREG(i->output_stat.st_mode) && !S_ISBLK(i->output_stat.st_mode)) + return log_error_errno(SYNTHETIC_ERRNO(EBADFD), + "Target file is not a regular file or block device"); + + if (i->offset != UINT64_MAX) { + if (lseek(i->output_fd, i->offset, SEEK_SET) == (off_t) -1) + return log_error_errno(errno, "Failed to seek to offset: %m"); + } + + return 0; +} + +static int raw_import_try_reflink(RawImport *i) { + off_t p; + int r; + + assert(i); + assert(i->input_fd >= 0); + assert(i->output_fd >= 0); + + if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED) + return 0; + + if (i->offset != UINT64_MAX || i->size_max != UINT64_MAX) + return 0; + + if (!S_ISREG(i->input_stat.st_mode) || !S_ISREG(i->output_stat.st_mode)) + return 0; + + p = lseek(i->input_fd, 0, SEEK_CUR); + if (p == (off_t) -1) + return log_error_errno(errno, "Failed to read file offset of input file: %m"); + + /* Let's only try a btrfs reflink, if we are reading from the beginning of the file */ + if ((uint64_t) p != (uint64_t) i->buffer_size) + return 0; + + r = btrfs_reflink(i->input_fd, i->output_fd); + if (r >= 0) + return 1; + + log_debug_errno(r, "Couldn't establish reflink, using copy: %m"); + return 0; +} + +static int raw_import_write(const void *p, size_t sz, void *userdata) { + RawImport *i = ASSERT_PTR(userdata); + bool too_much = false; + int r; + + assert(p); + assert(sz > 0); + + if (i->written_uncompressed >= UINT64_MAX - sz) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); + + if (i->size_max != UINT64_MAX) { + if (i->written_uncompressed >= i->size_max) { + too_much = true; + goto finish; + } + + if (i->written_uncompressed + sz > i->size_max) { + too_much = true; + sz = i->size_max - i->written_uncompressed; /* since we have the data in memory + * already, we might as well write it to + * disk to the max */ + } + } + + /* Generate sparse file if we created/truncated the file */ + if (S_ISREG(i->output_stat.st_mode) && i->offset == UINT64_MAX) { + ssize_t n; + + n = sparse_write(i->output_fd, p, sz, 64); + if (n < 0) + return log_error_errno((int) n, "Failed to write file: %m"); + if ((size_t) n < sz) + return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); + } else { + r = loop_write(i->output_fd, p, sz, false); + if (r < 0) + return log_error_errno(r, "Failed to write file: %m"); + } + + i->written_uncompressed += sz; + +finish: + if (too_much) + return log_error_errno(SYNTHETIC_ERRNO(E2BIG), "File too large"); + + return 0; +} + +static int raw_import_process(RawImport *i) { + ssize_t l; + int r; + + assert(i); + assert(i->buffer_size < sizeof(i->buffer)); + + l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to read input file: %m"); + goto finish; + } + + i->buffer_size += l; + + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + + if (l == 0) { /* EOF */ + log_debug("File too short to be compressed, as no compression signature fits in, thus assuming uncompressed."); + import_uncompress_force_off(&i->compress); + } else { + r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + if (r < 0) { + log_error_errno(r, "Failed to detect file compression: %m"); + goto finish; + } + if (r == 0) /* Need more data */ + return 0; + } + + r = raw_import_open_disk(i); + if (r < 0) + goto finish; + + r = raw_import_try_reflink(i); + if (r < 0) + goto finish; + if (r > 0) + goto complete; + } + + r = import_uncompress(&i->compress, i->buffer, i->buffer_size, raw_import_write, i); + if (r < 0) { + log_error_errno(r, "Failed to decode and write: %m"); + goto finish; + } + + i->written_compressed += i->buffer_size; + i->buffer_size = 0; + + if (l == 0) /* EOF */ + goto complete; + + raw_import_report_progress(i); + + return 0; + +complete: + r = raw_import_finish(i); + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); + + return 0; +} + +static int raw_import_on_input(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + RawImport *i = userdata; + + return raw_import_process(i); +} + +static int raw_import_on_defer(sd_event_source *s, void *userdata) { + RawImport *i = userdata; + + return raw_import_process(i); +} + +int raw_import_start( + RawImport *i, + int fd, + const char *local, + uint64_t offset, + uint64_t size_max, + ImportFlags flags) { + int r; + + assert(i); + assert(fd >= 0); + assert(local); + assert(!(flags & ~IMPORT_FLAGS_MASK_RAW)); + assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT)); + + if (!import_validate_local(local, flags)) + return -EINVAL; + + if (i->input_fd >= 0) + return -EBUSY; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + + i->flags = flags; + i->offset = offset; + i->size_max = size_max; + + if (fstat(fd, &i->input_stat) < 0) + return -errno; + + r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i); + if (r == -EPERM) { + /* This fd does not support epoll, for example because it is a regular file. Busy read in that case */ + r = sd_event_add_defer(i->event, &i->input_event_source, raw_import_on_defer, i); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(i->input_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + i->input_fd = fd; + return 0; +} diff --git a/src/import/import-raw.h b/src/import/import-raw.h new file mode 100644 index 0000000..63384eb --- /dev/null +++ b/src/import/import-raw.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "sd-event.h" + +#include "import-common.h" +#include "import-util.h" +#include "macro.h" + +typedef struct RawImport RawImport; + +typedef void (*RawImportFinished)(RawImport *import, int error, void *userdata); + +int raw_import_new(RawImport **import, sd_event *event, const char *image_root, RawImportFinished on_finished, void *userdata); +RawImport* raw_import_unref(RawImport *import); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref); + +int raw_import_start(RawImport *i, int fd, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags); diff --git a/src/import/import-tar.c b/src/import/import-tar.c new file mode 100644 index 0000000..f31d3d7 --- /dev/null +++ b/src/import/import-tar.c @@ -0,0 +1,381 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <linux/fs.h> + +#include "sd-daemon.h" +#include "sd-event.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "fd-util.h" +#include "fileio.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-compress.h" +#include "import-tar.h" +#include "install-file.h" +#include "io-util.h" +#include "machine-pool.h" +#include "mkdir-label.h" +#include "path-util.h" +#include "process-util.h" +#include "qcow2-util.h" +#include "ratelimit.h" +#include "rm-rf.h" +#include "string-util.h" +#include "tmpfile-util.h" +#include "util.h" + +struct TarImport { + sd_event *event; + + char *image_root; + + TarImportFinished on_finished; + void *userdata; + + char *local; + ImportFlags flags; + + char *temp_path; + char *final_path; + + int input_fd; + int tar_fd; + + ImportCompress compress; + + sd_event_source *input_event_source; + + uint8_t buffer[16*1024]; + size_t buffer_size; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + struct stat input_stat; + + pid_t tar_pid; + + unsigned last_percent; + RateLimit progress_ratelimit; +}; + +TarImport* tar_import_unref(TarImport *i) { + if (!i) + return NULL; + + sd_event_source_unref(i->input_event_source); + + if (i->tar_pid > 1) + sigkill_wait(i->tar_pid); + + rm_rf_subvolume_and_free(i->temp_path); + + import_compress_free(&i->compress); + + sd_event_unref(i->event); + + safe_close(i->tar_fd); + + free(i->final_path); + free(i->image_root); + free(i->local); + return mfree(i); +} + +int tar_import_new( + TarImport **ret, + sd_event *event, + const char *image_root, + TarImportFinished on_finished, + void *userdata) { + + _cleanup_(tar_import_unrefp) TarImport *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + i = new(TarImport, 1); + if (!i) + return -ENOMEM; + + *i = (TarImport) { + .input_fd = -1, + .tar_fd = -1, + .on_finished = on_finished, + .userdata = userdata, + .last_percent = UINT_MAX, + .image_root = TAKE_PTR(root), + .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 }, + }; + + if (event) + i->event = sd_event_ref(event); + else { + r = sd_event_default(&i->event); + if (r < 0) + return r; + } + + *ret = TAKE_PTR(i); + + return 0; +} + +static void tar_import_report_progress(TarImport *i) { + unsigned percent; + assert(i); + + /* We have no size information, unless the source is a regular file */ + if (!S_ISREG(i->input_stat.st_mode)) + return; + + if (i->written_compressed >= (uint64_t) i->input_stat.st_size) + percent = 100; + else + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size); + + if (percent == i->last_percent) + return; + + if (!ratelimit_below(&i->progress_ratelimit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Imported %u%%.", percent); + + i->last_percent = percent; +} + +static int tar_import_finish(TarImport *i) { + const char *d; + int r; + + assert(i); + assert(i->tar_fd >= 0); + + i->tar_fd = safe_close(i->tar_fd); + + if (i->tar_pid > 0) { + r = wait_for_terminate_and_check("tar", TAKE_PID(i->tar_pid), WAIT_LOG); + if (r < 0) + return r; + if (r != EXIT_SUCCESS) + return -EPROTO; + } + + assert_se(d = i->temp_path ?: i->local); + + r = import_mangle_os_tree(d); + if (r < 0) + return r; + + r = install_file( + AT_FDCWD, d, + AT_FDCWD, i->final_path, + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); + if (r < 0) + return log_error_errno(r, "Failed to move '%s' into place: %m", i->final_path ?: i->local); + + i->temp_path = mfree(i->temp_path); + + return 0; +} + +static int tar_import_fork_tar(TarImport *i) { + const char *d, *root; + int r; + + assert(i); + assert(i->local); + assert(!i->final_path); + assert(!i->temp_path); + assert(i->tar_fd < 0); + + if (i->flags & IMPORT_DIRECT) { + d = i->local; + root = NULL; + } else { + i->final_path = path_join(i->image_root, i->local); + if (!i->final_path) + return log_oom(); + + r = tempfn_random(i->final_path, NULL, &i->temp_path); + if (r < 0) + return log_oom(); + + d = i->temp_path; + root = i->image_root; + } + + assert(d); + + (void) mkdir_parents_label(d, 0700); + + if (FLAGS_SET(i->flags, IMPORT_DIRECT|IMPORT_FORCE)) + (void) rm_rf(d, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + if (i->flags & IMPORT_BTRFS_SUBVOL) + r = btrfs_subvol_make_fallback(d, 0755); + else + r = RET_NERRNO(mkdir(d, 0755)); + if (r == -EEXIST && (i->flags & IMPORT_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise, + * because in that case our temporary path collided */ + r = 0; + if (r < 0) + return log_error_errno(r, "Failed to create directory/subvolume %s: %m", d); + if (r > 0 && (i->flags & IMPORT_BTRFS_QUOTA)) { /* actually btrfs subvol */ + if (!(i->flags & IMPORT_DIRECT)) + (void) import_assign_pool_quota_and_warn(root); + (void) import_assign_pool_quota_and_warn(d); + } + + i->tar_fd = import_fork_tar_x(d, &i->tar_pid); + if (i->tar_fd < 0) + return i->tar_fd; + + return 0; +} + +static int tar_import_write(const void *p, size_t sz, void *userdata) { + TarImport *i = userdata; + int r; + + r = loop_write(i->tar_fd, p, sz, false); + if (r < 0) + return r; + + i->written_uncompressed += sz; + + return 0; +} + +static int tar_import_process(TarImport *i) { + ssize_t l; + int r; + + assert(i); + assert(i->buffer_size < sizeof(i->buffer)); + + l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to read input file: %m"); + goto finish; + } + + i->buffer_size += l; + + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + + if (l == 0) { /* EOF */ + log_debug("File too short to be compressed, as no compression signature fits in, thus assuming uncompressed."); + import_uncompress_force_off(&i->compress); + } else { + r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + if (r < 0) { + log_error_errno(r, "Failed to detect file compression: %m"); + goto finish; + } + if (r == 0) /* Need more data */ + return 0; + } + + r = tar_import_fork_tar(i); + if (r < 0) + goto finish; + } + + r = import_uncompress(&i->compress, i->buffer, i->buffer_size, tar_import_write, i); + if (r < 0) { + log_error_errno(r, "Failed to decode and write: %m"); + goto finish; + } + + i->written_compressed += i->buffer_size; + i->buffer_size = 0; + + if (l == 0) { /* EOF */ + r = tar_import_finish(i); + goto finish; + } + + tar_import_report_progress(i); + + return 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); + + return 0; +} + +static int tar_import_on_input(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + TarImport *i = userdata; + + return tar_import_process(i); +} + +static int tar_import_on_defer(sd_event_source *s, void *userdata) { + TarImport *i = userdata; + + return tar_import_process(i); +} + +int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags) { + int r; + + assert(i); + assert(fd >= 0); + assert(local); + assert(!(flags & ~IMPORT_FLAGS_MASK_TAR)); + + if (!import_validate_local(local, flags)) + return -EINVAL; + + if (i->input_fd >= 0) + return -EBUSY; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + + i->flags = flags; + + if (fstat(fd, &i->input_stat) < 0) + return -errno; + + r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i); + if (r == -EPERM) { + /* This fd does not support epoll, for example because it is a regular file. Busy read in that case */ + r = sd_event_add_defer(i->event, &i->input_event_source, tar_import_on_defer, i); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(i->input_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + i->input_fd = fd; + return 0; +} diff --git a/src/import/import-tar.h b/src/import/import-tar.h new file mode 100644 index 0000000..63b0bd4 --- /dev/null +++ b/src/import/import-tar.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "sd-event.h" + +#include "import-common.h" +#include "import-util.h" +#include "macro.h" + +typedef struct TarImport TarImport; + +typedef void (*TarImportFinished)(TarImport *import, int error, void *userdata); + +int tar_import_new(TarImport **import, sd_event *event, const char *image_root, TarImportFinished on_finished, void *userdata); +TarImport* tar_import_unref(TarImport *import); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarImport*, tar_import_unref); + +int tar_import_start(TarImport *import, int fd, const char *local, ImportFlags flags); diff --git a/src/import/import.c b/src/import/import.c new file mode 100644 index 0000000..c762124 --- /dev/null +++ b/src/import/import.c @@ -0,0 +1,491 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <getopt.h> +#include <locale.h> + +#include "sd-event.h" +#include "sd-id128.h" + +#include "alloc-util.h" +#include "discover-image.h" +#include "env-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-raw.h" +#include "import-tar.h" +#include "import-util.h" +#include "io-util.h" +#include "main-func.h" +#include "parse-argument.h" +#include "parse-util.h" +#include "signal-util.h" +#include "string-util.h" +#include "terminal-util.h" +#include "verbs.h" + +static const char *arg_image_root = "/var/lib/machines"; +static ImportFlags arg_import_flags = IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC; +static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; + +static int normalize_local(const char *local, char **ret) { + _cleanup_free_ char *ll = NULL; + int r; + + assert(ret); + + if (arg_import_flags & IMPORT_DIRECT) { + + if (!local) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified."); + + if (!path_is_absolute(local)) { + ll = path_join(arg_image_root, local); + if (!ll) + return log_oom(); + + local = ll; + } + + if (!path_is_valid(local)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local path name '%s' is not valid.", local); + } else { + if (local) { + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local image name '%s' is not valid.", + local); + } else + local = "imported"; + + if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) { + r = image_find(IMAGE_MACHINE, local, NULL, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else + return log_error_errno(SYNTHETIC_ERRNO(EEXIST), + "Image '%s' already exists.", + local); + } + } + + if (!ll) { + ll = strdup(local); + if (!ll) + return log_oom(); + } + + *ret = TAKE_PTR(ll); + return 0; +} + +static int open_source(const char *path, const char *local, int *ret_open_fd) { + _cleanup_close_ int open_fd = -1; + int retval; + + assert(local); + assert(ret_open_fd); + + if (path) { + open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open source file '%s': %m", path); + + retval = open_fd; + + if (arg_offset != UINT64_MAX) + log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", path, arg_offset, local); + else + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + retval = STDIN_FILENO; + + (void) fd_get_path(STDIN_FILENO, &pretty); + + if (arg_offset != UINT64_MAX) + log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", strempty(pretty), arg_offset, local); + else + log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); + } + + *ret_open_fd = TAKE_FD(open_fd); + return retval; +} + +static void on_tar_finished(TarImport *import, int error, void *userdata) { + sd_event *event = userdata; + assert(import); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int import_tar(int argc, char *argv[], void *userdata) { + _cleanup_(tar_import_unrefp) TarImport *import = NULL; + _cleanup_free_ char *ll = NULL, *normalized = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (argc >= 2) + path = empty_or_dash_to_null(argv[1]); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else if (path) { + _cleanup_free_ char *l = NULL; + + r = path_extract_filename(path, &l); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from path '%s': %m", path); + + r = tar_strip_suffixes(l, &ll); + if (r < 0) + return log_oom(); + + local = ll; + } + + r = normalize_local(local, &normalized); + if (r < 0) + return r; + + fd = open_source(path, normalized, &open_fd); + if (fd < 0) + return r; + + r = import_allocate_event_with_signals(&event); + if (r < 0) + return r; + + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) + log_info("File system synchronization on completion is off."); + + r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate importer: %m"); + + r = tar_import_start( + import, + fd, + normalized, + arg_import_flags & IMPORT_FLAGS_MASK_TAR); + if (r < 0) + return log_error_errno(r, "Failed to import image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawImport *import, int error, void *userdata) { + sd_event *event = userdata; + assert(import); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int import_raw(int argc, char *argv[], void *userdata) { + _cleanup_(raw_import_unrefp) RawImport *import = NULL; + _cleanup_free_ char *ll = NULL, *normalized = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (argc >= 2) + path = empty_or_dash_to_null(argv[1]); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else if (path) { + _cleanup_free_ char *l = NULL; + + r = path_extract_filename(path, &l); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from path '%s': %m", path); + + r = raw_strip_suffixes(l, &ll); + if (r < 0) + return log_oom(); + + local = ll; + } + + r = normalize_local(local, &normalized); + if (r < 0) + return r; + + fd = open_source(path, normalized, &open_fd); + if (fd < 0) + return fd; + + r = import_allocate_event_with_signals(&event); + if (r < 0) + return r; + + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) + log_info("File system synchronization on completion is off."); + + r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate importer: %m"); + + r = raw_import_start( + import, + fd, + normalized, + arg_offset, + arg_size_max, + arg_import_flags & IMPORT_FLAGS_MASK_RAW); + if (r < 0) + return log_error_errno(r, "Failed to import image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%1$s [OPTIONS...] {COMMAND} ...\n" + "\n%4$sImport container or virtual machine images.%5$s\n" + "\n%2$sCommands:%3$s\n" + " tar FILE [NAME] Import a TAR image\n" + " raw FILE [NAME] Import a RAW image\n" + "\n%2$sOptions:%3$s\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --force Force creation of image\n" + " --image-root=PATH Image root directory\n" + " --read-only Create a read-only image\n" + " --direct Import directly to specified file\n" + " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n" + " instead of a directory\n" + " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n" + " subvolume\n" + " --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n" + " regular disk images\n" + " --sync=BOOL Controls whether to sync() before completing\n" + " --offset=BYTES Offset to seek to in destination\n" + " --size-max=BYTES Maximum number of bytes to write to destination\n", + program_invocation_short_name, + ansi_underline(), + ansi_normal(), + ansi_highlight(), + ansi_normal()); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORCE, + ARG_IMAGE_ROOT, + ARG_READ_ONLY, + ARG_DIRECT, + ARG_BTRFS_SUBVOL, + ARG_BTRFS_QUOTA, + ARG_CONVERT_QCOW2, + ARG_SYNC, + ARG_OFFSET, + ARG_SIZE_MAX, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "force", no_argument, NULL, ARG_FORCE }, + { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "direct", no_argument, NULL, ARG_DIRECT }, + { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL }, + { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA }, + { "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 }, + { "sync", required_argument, NULL, ARG_SYNC }, + { "offset", required_argument, NULL, ARG_OFFSET }, + { "size-max", required_argument, NULL, ARG_SIZE_MAX }, + {} + }; + + int r, c; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORCE: + arg_import_flags |= IMPORT_FORCE; + break; + + case ARG_IMAGE_ROOT: + arg_image_root = optarg; + break; + + case ARG_READ_ONLY: + arg_import_flags |= IMPORT_READ_ONLY; + break; + + case ARG_DIRECT: + arg_import_flags |= IMPORT_DIRECT; + break; + + case ARG_BTRFS_SUBVOL: + r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r); + break; + + case ARG_BTRFS_QUOTA: + r = parse_boolean_argument("--btrfs-quota=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r); + break; + + case ARG_CONVERT_QCOW2: + r = parse_boolean_argument("--convert-qcow2=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_CONVERT_QCOW2, r); + break; + + case ARG_SYNC: + r = parse_boolean_argument("--sync=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_SYNC, r); + break; + + case ARG_OFFSET: { + uint64_t u; + + r = safe_atou64(optarg, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg); + + arg_offset = u; + break; + } + + case ARG_SIZE_MAX: { + uint64_t u; + + r = parse_size(optarg, 1024, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg); + + arg_size_max = u; + break; + } + + case '?': + return -EINVAL; + + default: + assert_not_reached(); + } + + /* Make sure offset+size is still in the valid range if both set */ + if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX && + ((arg_size_max > (UINT64_MAX - arg_offset)) || + !FILE_SIZE_VALID(arg_offset + arg_size_max))) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range."); + + if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); + + return 1; +} + +static int import_main(int argc, char *argv[]) { + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, import_tar }, + { "raw", 2, 3, 0, import_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +static void parse_env(void) { + int r; + + /* Let's make these relatively low-level settings also controllable via env vars. User can then set + * them to systemd-import if they like to tweak behaviour */ + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL"); + if (r >= 0) + SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA"); + if (r >= 0) + SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_SYNC"); + if (r >= 0) + SET_FLAG(arg_import_flags, IMPORT_SYNC, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m"); +} + +static int run(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + parse_env(); + + r = parse_argv(argc, argv); + if (r <= 0) + return r; + + (void) ignore_signals(SIGPIPE); + + return import_main(argc, argv); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/importd.c b/src/import/importd.c new file mode 100644 index 0000000..6692589 --- /dev/null +++ b/src/import/importd.c @@ -0,0 +1,1420 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <sys/prctl.h> +#include <sys/wait.h> + +#include "sd-bus.h" + +#include "alloc-util.h" +#include "bus-common-errors.h" +#include "bus-get-properties.h" +#include "bus-log-control-api.h" +#include "bus-polkit.h" +#include "def.h" +#include "env-util.h" +#include "fd-util.h" +#include "float.h" +#include "hostname-util.h" +#include "import-util.h" +#include "machine-pool.h" +#include "main-func.h" +#include "missing_capability.h" +#include "mkdir-label.h" +#include "parse-util.h" +#include "path-util.h" +#include "percent-util.h" +#include "process-util.h" +#include "service-util.h" +#include "signal-util.h" +#include "socket-util.h" +#include "stat-util.h" +#include "string-table.h" +#include "strv.h" +#include "syslog-util.h" +#include "user-util.h" +#include "util.h" +#include "web-util.h" + +typedef struct Transfer Transfer; +typedef struct Manager Manager; + +typedef enum TransferType { + TRANSFER_IMPORT_TAR, + TRANSFER_IMPORT_RAW, + TRANSFER_IMPORT_FS, + TRANSFER_EXPORT_TAR, + TRANSFER_EXPORT_RAW, + TRANSFER_PULL_TAR, + TRANSFER_PULL_RAW, + _TRANSFER_TYPE_MAX, + _TRANSFER_TYPE_INVALID = -EINVAL, +} TransferType; + +struct Transfer { + Manager *manager; + + uint32_t id; + char *object_path; + + TransferType type; + ImportVerify verify; + + char *remote; + char *local; + bool force_local; + bool read_only; + + char *format; + + pid_t pid; + + int log_fd; + + char log_message[LINE_MAX]; + size_t log_message_size; + + sd_event_source *pid_event_source; + sd_event_source *log_event_source; + + unsigned n_canceled; + unsigned progress_percent; + + int stdin_fd; + int stdout_fd; +}; + +struct Manager { + sd_event *event; + sd_bus *bus; + + uint32_t current_transfer_id; + Hashmap *transfers; + + Hashmap *polkit_registry; + + int notify_fd; + + sd_event_source *notify_event_source; + + bool use_btrfs_subvol; + bool use_btrfs_quota; +}; + +#define TRANSFERS_MAX 64 + +static const char* const transfer_type_table[_TRANSFER_TYPE_MAX] = { + [TRANSFER_IMPORT_TAR] = "import-tar", + [TRANSFER_IMPORT_RAW] = "import-raw", + [TRANSFER_IMPORT_FS] = "import-fs", + [TRANSFER_EXPORT_TAR] = "export-tar", + [TRANSFER_EXPORT_RAW] = "export-raw", + [TRANSFER_PULL_TAR] = "pull-tar", + [TRANSFER_PULL_RAW] = "pull-raw", +}; + +DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(transfer_type, TransferType); + +static Transfer *transfer_unref(Transfer *t) { + if (!t) + return NULL; + + if (t->manager) + hashmap_remove(t->manager->transfers, UINT32_TO_PTR(t->id)); + + sd_event_source_unref(t->pid_event_source); + sd_event_source_unref(t->log_event_source); + + free(t->remote); + free(t->local); + free(t->format); + free(t->object_path); + + if (t->pid > 1) + sigkill_wait(t->pid); + + safe_close(t->log_fd); + safe_close(t->stdin_fd); + safe_close(t->stdout_fd); + + return mfree(t); +} + +DEFINE_TRIVIAL_CLEANUP_FUNC(Transfer*, transfer_unref); + +static int transfer_new(Manager *m, Transfer **ret) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + uint32_t id; + int r; + + assert(m); + assert(ret); + + if (hashmap_size(m->transfers) >= TRANSFERS_MAX) + return -E2BIG; + + t = new(Transfer, 1); + if (!t) + return -ENOMEM; + + *t = (Transfer) { + .type = _TRANSFER_TYPE_INVALID, + .log_fd = -1, + .stdin_fd = -1, + .stdout_fd = -1, + .verify = _IMPORT_VERIFY_INVALID, + .progress_percent= UINT_MAX, + }; + + id = m->current_transfer_id + 1; + + if (asprintf(&t->object_path, "/org/freedesktop/import1/transfer/_%" PRIu32, id) < 0) + return -ENOMEM; + + r = hashmap_ensure_put(&m->transfers, &trivial_hash_ops, UINT32_TO_PTR(id), t); + if (r < 0) + return r; + + m->current_transfer_id = id; + + t->manager = m; + t->id = id; + + *ret = TAKE_PTR(t); + + return 0; +} + +static double transfer_percent_as_double(Transfer *t) { + assert(t); + + if (t->progress_percent == UINT_MAX) + return -DBL_MAX; + + return (double) t->progress_percent / 100.0; +} + +static void transfer_send_log_line(Transfer *t, const char *line) { + int r, priority = LOG_INFO; + + assert(t); + assert(line); + + syslog_parse_priority(&line, &priority, true); + + log_full(priority, "(transfer%" PRIu32 ") %s", t->id, line); + + r = sd_bus_emit_signal( + t->manager->bus, + t->object_path, + "org.freedesktop.import1.Transfer", + "LogMessage", + "us", + priority, + line); + if (r < 0) + log_warning_errno(r, "Cannot emit log message signal, ignoring: %m"); + } + +static void transfer_send_logs(Transfer *t, bool flush) { + assert(t); + + /* Try to send out all log messages, if we can. But if we + * can't we remove the messages from the buffer, but don't + * fail */ + + while (t->log_message_size > 0) { + _cleanup_free_ char *n = NULL; + char *e; + + if (t->log_message_size >= sizeof(t->log_message)) + e = t->log_message + sizeof(t->log_message); + else { + char *a, *b; + + a = memchr(t->log_message, 0, t->log_message_size); + b = memchr(t->log_message, '\n', t->log_message_size); + + if (a && b) + e = a < b ? a : b; + else if (a) + e = a; + else + e = b; + } + + if (!e) { + if (!flush) + return; + + e = t->log_message + t->log_message_size; + } + + n = strndup(t->log_message, e - t->log_message); + + /* Skip over NUL and newlines */ + while (e < t->log_message + t->log_message_size && IN_SET(*e, 0, '\n')) + e++; + + memmove(t->log_message, e, t->log_message + sizeof(t->log_message) - e); + t->log_message_size -= e - t->log_message; + + if (!n) { + log_oom(); + continue; + } + + if (isempty(n)) + continue; + + transfer_send_log_line(t, n); + } +} + +static int transfer_finalize(Transfer *t, bool success) { + int r; + + assert(t); + + transfer_send_logs(t, true); + + r = sd_bus_emit_signal( + t->manager->bus, + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "TransferRemoved", + "uos", + t->id, + t->object_path, + success ? "done" : + t->n_canceled > 0 ? "canceled" : "failed"); + + if (r < 0) + log_error_errno(r, "Cannot emit message: %m"); + + transfer_unref(t); + return 0; +} + +static int transfer_cancel(Transfer *t) { + int r; + + assert(t); + + r = kill_and_sigcont(t->pid, t->n_canceled < 3 ? SIGTERM : SIGKILL); + if (r < 0) + return r; + + t->n_canceled++; + return 0; +} + +static int transfer_on_pid(sd_event_source *s, const siginfo_t *si, void *userdata) { + Transfer *t = ASSERT_PTR(userdata); + bool success = false; + + assert(s); + + if (si->si_code == CLD_EXITED) { + if (si->si_status != 0) + log_error("Transfer process failed with exit code %i.", si->si_status); + else { + log_debug("Transfer process succeeded."); + success = true; + } + + } else if (IN_SET(si->si_code, CLD_KILLED, CLD_DUMPED)) + log_error("Transfer process terminated by signal %s.", signal_to_string(si->si_status)); + else + log_error("Transfer process failed due to unknown reason."); + + t->pid = 0; + + return transfer_finalize(t, success); +} + +static int transfer_on_log(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + Transfer *t = ASSERT_PTR(userdata); + ssize_t l; + + assert(s); + + l = read(fd, t->log_message + t->log_message_size, sizeof(t->log_message) - t->log_message_size); + if (l < 0) + log_error_errno(errno, "Failed to read log message: %m"); + if (l <= 0) { + /* EOF/read error. We just close the pipe here, and + * close the watch, waiting for the SIGCHLD to arrive, + * before we do anything else. */ + t->log_event_source = sd_event_source_unref(t->log_event_source); + return 0; + } + + t->log_message_size += l; + + transfer_send_logs(t, false); + + return 0; +} + +static int transfer_start(Transfer *t) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + int r; + + assert(t); + assert(t->pid <= 0); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return -errno; + + r = safe_fork("(sd-transfer)", FORK_RESET_SIGNALS|FORK_DEATHSIG, &t->pid); + if (r < 0) + return r; + if (r == 0) { + const char *cmd[] = { + NULL, /* systemd-import, systemd-import-fs, systemd-export or systemd-pull */ + NULL, /* tar, raw */ + NULL, /* --verify= */ + NULL, /* verify argument */ + NULL, /* maybe --force */ + NULL, /* maybe --read-only */ + NULL, /* if so: the actual URL */ + NULL, /* maybe --format= */ + NULL, /* if so: the actual format */ + NULL, /* remote */ + NULL, /* local */ + NULL + }; + unsigned k = 0; + + /* Child */ + + pipefd[0] = safe_close(pipefd[0]); + + r = rearrange_stdio(TAKE_FD(t->stdin_fd), + t->stdout_fd < 0 ? pipefd[1] : TAKE_FD(t->stdout_fd), + pipefd[1]); + TAKE_FD(pipefd[1]); + if (r < 0) { + log_error_errno(r, "Failed to set stdin/stdout/stderr: %m"); + _exit(EXIT_FAILURE); + } + + if (setenv("SYSTEMD_LOG_TARGET", "console-prefixed", 1) < 0 || + setenv("NOTIFY_SOCKET", "/run/systemd/import/notify", 1) < 0) { + log_error_errno(errno, "setenv() failed: %m"); + _exit(EXIT_FAILURE); + } + + r = setenv_systemd_exec_pid(true); + if (r < 0) + log_warning_errno(r, "Failed to update $SYSTEMD_EXEC_PID, ignoring: %m"); + + switch (t->type) { + + case TRANSFER_IMPORT_TAR: + case TRANSFER_IMPORT_RAW: + cmd[k++] = SYSTEMD_IMPORT_PATH; + break; + + case TRANSFER_IMPORT_FS: + cmd[k++] = SYSTEMD_IMPORT_FS_PATH; + break; + + case TRANSFER_EXPORT_TAR: + case TRANSFER_EXPORT_RAW: + cmd[k++] = SYSTEMD_EXPORT_PATH; + break; + + case TRANSFER_PULL_TAR: + case TRANSFER_PULL_RAW: + cmd[k++] = SYSTEMD_PULL_PATH; + break; + + default: + assert_not_reached(); + } + + switch (t->type) { + + case TRANSFER_IMPORT_TAR: + case TRANSFER_EXPORT_TAR: + case TRANSFER_PULL_TAR: + cmd[k++] = "tar"; + break; + + case TRANSFER_IMPORT_RAW: + case TRANSFER_EXPORT_RAW: + case TRANSFER_PULL_RAW: + cmd[k++] = "raw"; + break; + + case TRANSFER_IMPORT_FS: + cmd[k++] = "run"; + break; + + default: + break; + } + + if (t->verify != _IMPORT_VERIFY_INVALID) { + cmd[k++] = "--verify"; + cmd[k++] = import_verify_to_string(t->verify); + } + + if (t->force_local) + cmd[k++] = "--force"; + if (t->read_only) + cmd[k++] = "--read-only"; + + if (t->format) { + cmd[k++] = "--format"; + cmd[k++] = t->format; + } + + if (!IN_SET(t->type, TRANSFER_EXPORT_TAR, TRANSFER_EXPORT_RAW)) { + if (t->remote) + cmd[k++] = t->remote; + else + cmd[k++] = "-"; + } + + if (t->local) + cmd[k++] = t->local; + cmd[k] = NULL; + + execv(cmd[0], (char * const *) cmd); + log_error_errno(errno, "Failed to execute %s tool: %m", cmd[0]); + _exit(EXIT_FAILURE); + } + + pipefd[1] = safe_close(pipefd[1]); + t->log_fd = TAKE_FD(pipefd[0]); + + t->stdin_fd = safe_close(t->stdin_fd); + + r = sd_event_add_child(t->manager->event, &t->pid_event_source, + t->pid, WEXITED, transfer_on_pid, t); + if (r < 0) + return r; + + r = sd_event_add_io(t->manager->event, &t->log_event_source, + t->log_fd, EPOLLIN, transfer_on_log, t); + if (r < 0) + return r; + + /* Make sure always process logging before SIGCHLD */ + r = sd_event_source_set_priority(t->log_event_source, SD_EVENT_PRIORITY_NORMAL -5); + if (r < 0) + return r; + + r = sd_bus_emit_signal( + t->manager->bus, + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "TransferNew", + "uo", + t->id, + t->object_path); + if (r < 0) + return r; + + return 0; +} + +static Manager *manager_unref(Manager *m) { + Transfer *t; + + if (!m) + return NULL; + + sd_event_source_unref(m->notify_event_source); + safe_close(m->notify_fd); + + while ((t = hashmap_first(m->transfers))) + transfer_unref(t); + + hashmap_free(m->transfers); + + bus_verify_polkit_async_registry_free(m->polkit_registry); + + m->bus = sd_bus_flush_close_unref(m->bus); + sd_event_unref(m->event); + + return mfree(m); +} + +DEFINE_TRIVIAL_CLEANUP_FUNC(Manager*, manager_unref); + +static int manager_on_notify(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + + char buf[NOTIFY_BUFFER_MAX+1]; + struct iovec iovec = { + .iov_base = buf, + .iov_len = sizeof(buf)-1, + }; + CMSG_BUFFER_TYPE(CMSG_SPACE(sizeof(struct ucred)) + + CMSG_SPACE(sizeof(int) * NOTIFY_FD_MAX)) control; + struct msghdr msghdr = { + .msg_iov = &iovec, + .msg_iovlen = 1, + .msg_control = &control, + .msg_controllen = sizeof(control), + }; + struct ucred *ucred; + Manager *m = userdata; + char *p, *e; + Transfer *t; + ssize_t n; + int r; + + n = recvmsg_safe(fd, &msghdr, MSG_DONTWAIT|MSG_CMSG_CLOEXEC); + if (n < 0) { + if (ERRNO_IS_TRANSIENT(n)) + return 0; + return (int) n; + } + + cmsg_close_all(&msghdr); + + if (msghdr.msg_flags & MSG_TRUNC) { + log_warning("Got overly long notification datagram, ignoring."); + return 0; + } + + ucred = CMSG_FIND_DATA(&msghdr, SOL_SOCKET, SCM_CREDENTIALS, struct ucred); + if (!ucred || ucred->pid <= 0) { + log_warning("Got notification datagram lacking credential information, ignoring."); + return 0; + } + + HASHMAP_FOREACH(t, m->transfers) + if (ucred->pid == t->pid) + break; + + if (!t) { + log_warning("Got notification datagram from unexpected peer, ignoring."); + return 0; + } + + buf[n] = 0; + + p = startswith(buf, "X_IMPORT_PROGRESS="); + if (!p) { + p = strstr(buf, "\nX_IMPORT_PROGRESS="); + if (!p) + return 0; + + p += 19; + } + + e = strchrnul(p, '\n'); + *e = 0; + + r = parse_percent(p); + if (r < 0) { + log_warning("Got invalid percent value, ignoring."); + return 0; + } + + t->progress_percent = (unsigned) r; + + log_debug("Got percentage from client: %u%%", t->progress_percent); + return 0; +} + +static int manager_new(Manager **ret) { + _cleanup_(manager_unrefp) Manager *m = NULL; + static const union sockaddr_union sa = { + .un.sun_family = AF_UNIX, + .un.sun_path = "/run/systemd/import/notify", + }; + int r; + + assert(ret); + + m = new(Manager, 1); + if (!m) + return -ENOMEM; + + *m = (Manager) { + .use_btrfs_subvol = true, + .use_btrfs_quota = true, + }; + + r = sd_event_default(&m->event); + if (r < 0) + return r; + + sd_event_set_watchdog(m->event, true); + + r = sd_bus_default_system(&m->bus); + if (r < 0) + return r; + + m->notify_fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); + if (m->notify_fd < 0) + return -errno; + + (void) mkdir_parents_label(sa.un.sun_path, 0755); + (void) sockaddr_un_unlink(&sa.un); + + if (bind(m->notify_fd, &sa.sa, SOCKADDR_UN_LEN(sa.un)) < 0) + return -errno; + + r = setsockopt_int(m->notify_fd, SOL_SOCKET, SO_PASSCRED, true); + if (r < 0) + return r; + + r = sd_event_add_io(m->event, &m->notify_event_source, + m->notify_fd, EPOLLIN, manager_on_notify, m); + if (r < 0) + return r; + + *ret = TAKE_PTR(m); + + return 0; +} + +static Transfer *manager_find(Manager *m, TransferType type, const char *remote) { + Transfer *t; + + assert(m); + assert(type >= 0); + assert(type < _TRANSFER_TYPE_MAX); + + HASHMAP_FOREACH(t, m->transfers) + if (t->type == type && streq_ptr(t->remote, remote)) + return t; + + return NULL; +} + +static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, force, read_only, r; + const char *local, *object; + Manager *m = ASSERT_PTR(userdata); + TransferType type; + struct stat st; + uint32_t id; + + assert(msg); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.import", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); + if (r < 0) + return r; + + if (fstat(fd, &st) < 0) + return -errno; + + if (!S_ISREG(st.st_mode) && !S_ISFIFO(st.st_mode)) + return -EINVAL; + + if (!hostname_is_valid(local, 0)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Local name %s is invalid", local); + + r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); + if (r < 0) + return r; + + type = streq_ptr(sd_bus_message_get_member(msg), "ImportTar") ? + TRANSFER_IMPORT_TAR : TRANSFER_IMPORT_RAW; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + t->force_local = force; + t->read_only = read_only; + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdin_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdin_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, force, read_only, r; + const char *local, *object; + Manager *m = ASSERT_PTR(userdata); + uint32_t id; + + assert(msg); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.import", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); + if (r < 0) + return r; + + r = fd_verify_directory(fd); + if (r < 0) + return r; + + if (!hostname_is_valid(local, 0)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Local name %s is invalid", local); + + r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); + if (r < 0) + return r; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = TRANSFER_IMPORT_FS; + t->force_local = force; + t->read_only = read_only; + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdin_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdin_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, r; + const char *local, *object, *format; + Manager *m = ASSERT_PTR(userdata); + TransferType type; + struct stat st; + uint32_t id; + + assert(msg); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.export", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "shs", &local, &fd, &format); + if (r < 0) + return r; + + if (!hostname_is_valid(local, 0)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Local name %s is invalid", local); + + if (fstat(fd, &st) < 0) + return -errno; + + if (!S_ISREG(st.st_mode) && !S_ISFIFO(st.st_mode)) + return -EINVAL; + + type = streq_ptr(sd_bus_message_get_member(msg), "ExportTar") ? + TRANSFER_EXPORT_TAR : TRANSFER_EXPORT_RAW; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + + if (!isempty(format)) { + t->format = strdup(format); + if (!t->format) + return -ENOMEM; + } + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdout_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdout_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + const char *remote, *local, *verify, *object; + Manager *m = ASSERT_PTR(userdata); + ImportVerify v; + TransferType type; + int force, r; + uint32_t id; + + assert(msg); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.pull", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "sssb", &remote, &local, &verify, &force); + if (r < 0) + return r; + + if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "URL %s is invalid", remote); + + if (isempty(local)) + local = NULL; + else if (!hostname_is_valid(local, 0)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Local name %s is invalid", local); + + if (isempty(verify)) + v = IMPORT_VERIFY_SIGNATURE; + else + v = import_verify_from_string(verify); + if (v < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Unknown verification mode %s", verify); + + r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); + if (r < 0) + return r; + + type = streq_ptr(sd_bus_message_get_member(msg), "PullTar") ? + TRANSFER_PULL_TAR : TRANSFER_PULL_RAW; + + if (manager_find(m, type, remote)) + return sd_bus_error_setf(error, BUS_ERROR_TRANSFER_IN_PROGRESS, + "Transfer for %s already in progress.", remote); + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + t->verify = v; + t->force_local = force; + + t->remote = strdup(remote); + if (!t->remote) + return -ENOMEM; + + if (local) { + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + } + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + +static int method_list_transfers(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; + Manager *m = ASSERT_PTR(userdata); + Transfer *t; + int r; + + assert(msg); + + r = sd_bus_message_new_method_return(msg, &reply); + if (r < 0) + return r; + + r = sd_bus_message_open_container(reply, 'a', "(usssdo)"); + if (r < 0) + return r; + + HASHMAP_FOREACH(t, m->transfers) { + + r = sd_bus_message_append( + reply, + "(usssdo)", + t->id, + transfer_type_to_string(t->type), + t->remote, + t->local, + transfer_percent_as_double(t), + t->object_path); + if (r < 0) + return r; + } + + r = sd_bus_message_close_container(reply); + if (r < 0) + return r; + + return sd_bus_send(NULL, reply, NULL); +} + +static int method_cancel(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + Transfer *t = ASSERT_PTR(userdata); + int r; + + assert(msg); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.pull", + NULL, + false, + UID_INVALID, + &t->manager->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = transfer_cancel(t); + if (r < 0) + return r; + + return sd_bus_reply_method_return(msg, NULL); +} + +static int method_cancel_transfer(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + Manager *m = ASSERT_PTR(userdata); + Transfer *t; + uint32_t id; + int r; + + assert(msg); + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.pull", + NULL, + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "u", &id); + if (r < 0) + return r; + if (id <= 0) + return sd_bus_error_set(error, SD_BUS_ERROR_INVALID_ARGS, "Invalid transfer id"); + + t = hashmap_get(m->transfers, UINT32_TO_PTR(id)); + if (!t) + return sd_bus_error_setf(error, BUS_ERROR_NO_SUCH_TRANSFER, "No transfer by id %" PRIu32, id); + + r = transfer_cancel(t); + if (r < 0) + return r; + + return sd_bus_reply_method_return(msg, NULL); +} + +static int property_get_progress( + sd_bus *bus, + const char *path, + const char *interface, + const char *property, + sd_bus_message *reply, + void *userdata, + sd_bus_error *error) { + + Transfer *t = ASSERT_PTR(userdata); + + assert(bus); + assert(reply); + + return sd_bus_message_append(reply, "d", transfer_percent_as_double(t)); +} + +static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_type, transfer_type, TransferType); +static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_verify, import_verify, ImportVerify); + +static int transfer_object_find( + sd_bus *bus, + const char *path, + const char *interface, + void *userdata, + void **found, + sd_bus_error *error) { + + Manager *m = ASSERT_PTR(userdata); + Transfer *t; + const char *p; + uint32_t id; + int r; + + assert(bus); + assert(path); + assert(interface); + assert(found); + + p = startswith(path, "/org/freedesktop/import1/transfer/_"); + if (!p) + return 0; + + r = safe_atou32(p, &id); + if (r < 0 || id == 0) + return 0; + + t = hashmap_get(m->transfers, UINT32_TO_PTR(id)); + if (!t) + return 0; + + *found = t; + return 1; +} + +static int transfer_node_enumerator( + sd_bus *bus, + const char *path, + void *userdata, + char ***nodes, + sd_bus_error *error) { + + _cleanup_strv_free_ char **l = NULL; + Manager *m = userdata; + Transfer *t; + unsigned k = 0; + + l = new0(char*, hashmap_size(m->transfers) + 1); + if (!l) + return -ENOMEM; + + HASHMAP_FOREACH(t, m->transfers) { + + l[k] = strdup(t->object_path); + if (!l[k]) + return -ENOMEM; + + k++; + } + + *nodes = TAKE_PTR(l); + + return 1; +} + +static const sd_bus_vtable transfer_vtable[] = { + SD_BUS_VTABLE_START(0), + + SD_BUS_PROPERTY("Id", "u", NULL, offsetof(Transfer, id), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Local", "s", NULL, offsetof(Transfer, local), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Remote", "s", NULL, offsetof(Transfer, remote), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Type", "s", property_get_type, offsetof(Transfer, type), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Verify", "s", property_get_verify, offsetof(Transfer, verify), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("Progress", "d", property_get_progress, 0, 0), + + SD_BUS_METHOD("Cancel", NULL, NULL, method_cancel, SD_BUS_VTABLE_UNPRIVILEGED), + + SD_BUS_SIGNAL_WITH_NAMES("LogMessage", + "us", + SD_BUS_PARAM(priority) + SD_BUS_PARAM(line), + 0), + + SD_BUS_VTABLE_END, +}; + +static const BusObjectImplementation transfer_object = { + "/org/freedesktop/import1/transfer", + "org.freedesktop.import1.Transfer", + .fallback_vtables = BUS_FALLBACK_VTABLES({transfer_vtable, transfer_object_find}), + .node_enumerator = transfer_node_enumerator, +}; + +static const sd_bus_vtable manager_vtable[] = { + SD_BUS_VTABLE_START(0), + + SD_BUS_METHOD_WITH_NAMES("ImportTar", + "hsbb", + SD_BUS_PARAM(fd) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(force) + SD_BUS_PARAM(read_only), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_import_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ImportRaw", + "hsbb", + SD_BUS_PARAM(fd) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(force) + SD_BUS_PARAM(read_only), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_import_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ImportFileSystem", + "hsbb", + SD_BUS_PARAM(fd) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(force) + SD_BUS_PARAM(read_only), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_import_fs, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ExportTar", + "shs", + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(fd) + SD_BUS_PARAM(format), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_export_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ExportRaw", + "shs", + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(fd) + SD_BUS_PARAM(format), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_export_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("PullTar", + "sssb", + SD_BUS_PARAM(url) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(verify_mode) + SD_BUS_PARAM(force), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_pull_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("PullRaw", + "sssb", + SD_BUS_PARAM(url) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(verify_mode) + SD_BUS_PARAM(force), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_pull_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ListTransfers", + NULL,, + "a(usssdo)", + SD_BUS_PARAM(transfers), + method_list_transfers, + SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("CancelTransfer", + "u", + SD_BUS_PARAM(transfer_id), + NULL,, + method_cancel_transfer, + SD_BUS_VTABLE_UNPRIVILEGED), + + SD_BUS_SIGNAL_WITH_NAMES("TransferNew", + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + 0), + SD_BUS_SIGNAL_WITH_NAMES("TransferRemoved", + "uos", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path) + SD_BUS_PARAM(result), + 0), + + SD_BUS_VTABLE_END, +}; + +static const BusObjectImplementation manager_object = { + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + .vtables = BUS_VTABLES(manager_vtable), + .children = BUS_IMPLEMENTATIONS(&transfer_object), +}; + +static int manager_add_bus_objects(Manager *m) { + int r; + + assert(m); + + r = bus_add_implementation(m->bus, &manager_object, m); + if (r < 0) + return r; + + r = bus_log_control_api_register(m->bus); + if (r < 0) + return r; + + r = sd_bus_request_name_async(m->bus, NULL, "org.freedesktop.import1", 0, NULL, NULL); + if (r < 0) + return log_error_errno(r, "Failed to request name: %m"); + + r = sd_bus_attach_event(m->bus, m->event, 0); + if (r < 0) + return log_error_errno(r, "Failed to attach bus to event loop: %m"); + + return 0; +} + +static bool manager_check_idle(void *userdata) { + Manager *m = userdata; + + return hashmap_isempty(m->transfers); +} + +static int manager_run(Manager *m) { + assert(m); + + return bus_event_loop_with_idle( + m->event, + m->bus, + "org.freedesktop.import1", + DEFAULT_EXIT_USEC, + manager_check_idle, + m); +} + +static void manager_parse_env(Manager *m) { + int r; + + assert(m); + + /* Same as src/import/{import,pull}.c: + * Let's make these relatively low-level settings also controllable via env vars. User can then set + * them for systemd-importd.service if they like to tweak behaviour */ + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL"); + if (r >= 0) + m->use_btrfs_subvol = r; + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA"); + if (r >= 0) + m->use_btrfs_quota = r; + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m"); +} + +static int run(int argc, char *argv[]) { + _cleanup_(manager_unrefp) Manager *m = NULL; + int r; + + log_setup(); + + r = service_parse_argv("systemd-importd.service", + "VM and container image import and export service.", + BUS_IMPLEMENTATIONS(&manager_object, + &log_control_object), + argc, argv); + if (r <= 0) + return r; + + umask(0022); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGCHLD, -1) >= 0); + + r = manager_new(&m); + if (r < 0) + return log_error_errno(r, "Failed to allocate manager object: %m"); + + manager_parse_env(m); + + r = manager_add_bus_objects(m); + if (r < 0) + return r; + + r = manager_run(m); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + return 0; +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/meson.build b/src/import/meson.build new file mode 100644 index 0000000..7e92307 --- /dev/null +++ b/src/import/meson.build @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +systemd_importd_sources = files( + 'importd.c') + +systemd_pull_sources = files( + 'pull.c', + 'pull-raw.c', + 'pull-raw.h', + 'pull-tar.c', + 'pull-tar.h', + 'pull-job.c', + 'pull-job.h', + 'pull-common.c', + 'pull-common.h', + 'curl-util.c', + 'curl-util.h') + +systemd_import_sources = files( + 'import.c', + 'import-raw.c', + 'import-raw.h', + 'import-tar.c', + 'import-tar.h') + +systemd_import_fs_sources = files( + 'import-fs.c') + +systemd_export_sources = files( + 'export.c', + 'export-tar.c', + 'export-tar.h', + 'export-raw.c', + 'export-raw.h') + +if conf.get('ENABLE_IMPORTD') == 1 + lib_import_common = static_library( + 'import-common', + 'import-common.c', + 'import-common.h', + 'import-compress.c', + 'import-compress.h', + 'qcow2-util.c', + 'qcow2-util.h', + include_directories : includes, + dependencies : [libbzip2, + libxz, + libz], + build_by_default : false) + + install_data('org.freedesktop.import1.conf', + install_dir : dbuspolicydir) + install_data('org.freedesktop.import1.service', + install_dir : dbussystemservicedir) + install_data('org.freedesktop.import1.policy', + install_dir : polkitpolicydir) + + install_data('import-pubring.gpg', + install_dir : rootlibexecdir) + # TODO: shouldn't this be in pkgdatadir? +endif + +tests += [ + [files('test-qcow2.c', + 'qcow2-util.c', + 'qcow2-util.h'), + [], + [libz], + [], 'HAVE_ZLIB', 'manual'], +] diff --git a/src/import/org.freedesktop.import1.conf b/src/import/org.freedesktop.import1.conf new file mode 100644 index 0000000..d252ff6 --- /dev/null +++ b/src/import/org.freedesktop.import1.conf @@ -0,0 +1,84 @@ +<?xml version="1.0"?> <!--*-nxml-*--> +<!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN" + "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd"> + +<!-- + SPDX-License-Identifier: LGPL-2.1-or-later + + This file is part of systemd. + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. +--> + +<busconfig> + + <policy user="root"> + <allow own="org.freedesktop.import1"/> + <allow send_destination="org.freedesktop.import1"/> + <allow receive_sender="org.freedesktop.import1"/> + </policy> + + <policy context="default"> + <deny send_destination="org.freedesktop.import1"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Introspectable"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Peer"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Properties" + send_member="Get"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.DBus.Properties" + send_member="GetAll"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ListTransfers"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="CancelTransfer"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ImportTar"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ImportRaw"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ImportFileSystem"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ExportTar"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="ExportRaw"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="PullTar"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Manager" + send_member="PullRaw"/> + + <allow send_destination="org.freedesktop.import1" + send_interface="org.freedesktop.import1.Transfer" + send_member="Cancel"/> + + <allow receive_sender="org.freedesktop.import1"/> + </policy> + +</busconfig> diff --git a/src/import/org.freedesktop.import1.policy b/src/import/org.freedesktop.import1.policy new file mode 100644 index 0000000..88e436d --- /dev/null +++ b/src/import/org.freedesktop.import1.policy @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> <!--*-nxml-*--> +<!DOCTYPE policyconfig PUBLIC "-//freedesktop//DTD PolicyKit Policy Configuration 1.0//EN" + "https://www.freedesktop.org/standards/PolicyKit/1/policyconfig.dtd"> + +<!-- + SPDX-License-Identifier: LGPL-2.1-or-later + + This file is part of systemd. + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. +--> + +<policyconfig> + + <vendor>The systemd Project</vendor> + <vendor_url>https://systemd.io</vendor_url> + + <action id="org.freedesktop.import1.import"> + <description gettext-domain="systemd">Import a VM or container image</description> + <message gettext-domain="systemd">Authentication is required to import a VM or container image</message> + <defaults> + <allow_any>auth_admin</allow_any> + <allow_inactive>auth_admin</allow_inactive> + <allow_active>auth_admin_keep</allow_active> + </defaults> + </action> + + <action id="org.freedesktop.import1.export"> + <description gettext-domain="systemd">Export a VM or container image</description> + <message gettext-domain="systemd">Authentication is required to export a VM or container image</message> + <defaults> + <allow_any>auth_admin</allow_any> + <allow_inactive>auth_admin</allow_inactive> + <allow_active>auth_admin_keep</allow_active> + </defaults> + </action> + + <action id="org.freedesktop.import1.pull"> + <description gettext-domain="systemd">Download a VM or container image</description> + <message gettext-domain="systemd">Authentication is required to download a VM or container image</message> + <defaults> + <allow_any>auth_admin</allow_any> + <allow_inactive>auth_admin</allow_inactive> + <allow_active>auth_admin_keep</allow_active> + </defaults> + </action> + +</policyconfig> diff --git a/src/import/org.freedesktop.import1.service b/src/import/org.freedesktop.import1.service new file mode 100644 index 0000000..4fe921f --- /dev/null +++ b/src/import/org.freedesktop.import1.service @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[D-BUS Service] +Name=org.freedesktop.import1 +Exec=/bin/false +User=root +SystemdService=dbus-org.freedesktop.import1.service diff --git a/src/import/pull-common.c b/src/import/pull-common.c new file mode 100644 index 0000000..028bae8 --- /dev/null +++ b/src/import/pull-common.c @@ -0,0 +1,677 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <sys/prctl.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "capability-util.h" +#include "copy.h" +#include "dirent-util.h" +#include "escape.h" +#include "fd-util.h" +#include "hostname-util.h" +#include "io-util.h" +#include "memory-util.h" +#include "path-util.h" +#include "process-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "rlimit-util.h" +#include "rm-rf.h" +#include "signal-util.h" +#include "siphash24.h" +#include "string-util.h" +#include "strv.h" +#include "util.h" +#include "web-util.h" + +#define FILENAME_ESCAPE "/.#\"\'" +#define HASH_URL_THRESHOLD_LENGTH (_POSIX_PATH_MAX - 16) + +int pull_find_old_etags( + const char *url, + const char *image_root, + int dt, + const char *prefix, + const char *suffix, + char ***etags) { + + int r; + + assert(url); + assert(etags); + + if (!image_root) + image_root = "/var/lib/machines"; + + _cleanup_free_ char *escaped_url = xescape(url, FILENAME_ESCAPE); + if (!escaped_url) + return -ENOMEM; + + _cleanup_closedir_ DIR *d = opendir(image_root); + if (!d) { + if (errno == ENOENT) { + *etags = NULL; + return 0; + } + + return -errno; + } + + _cleanup_strv_free_ char **ans = NULL; + + FOREACH_DIRENT_ALL(de, d, return -errno) { + _cleanup_free_ char *u = NULL; + const char *a, *b; + + if (de->d_type != DT_UNKNOWN && + de->d_type != dt) + continue; + + if (prefix) { + a = startswith(de->d_name, prefix); + if (!a) + continue; + } else + a = de->d_name; + + a = startswith(a, escaped_url); + if (!a) + continue; + + a = startswith(a, "."); + if (!a) + continue; + + if (suffix) { + b = endswith(de->d_name, suffix); + if (!b) + continue; + } else + b = strchr(de->d_name, 0); + + if (a >= b) + continue; + + ssize_t l = cunescape_length(a, b - a, 0, &u); + if (l < 0) { + assert(l >= INT8_MIN); + return l; + } + + if (!http_etag_is_valid(u)) + continue; + + r = strv_consume(&ans, TAKE_PTR(u)); + if (r < 0) + return r; + } + + *etags = TAKE_PTR(ans); + + return 0; +} + +static int hash_url(const char *url, char **ret) { + uint64_t h; + static const sd_id128_t k = SD_ID128_ARRAY(df,89,16,87,01,cc,42,30,98,ab,4a,19,a6,a5,63,4f); + + assert(url); + + h = siphash24(url, strlen(url), k.bytes); + if (asprintf(ret, "%"PRIx64, h) < 0) + return -ENOMEM; + + return 0; +} + +int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret) { + _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL; + char *path; + + assert(url); + assert(ret); + + if (!image_root) + image_root = "/var/lib/machines"; + + escaped_url = xescape(url, FILENAME_ESCAPE); + if (!escaped_url) + return -ENOMEM; + + if (etag) { + escaped_etag = xescape(etag, FILENAME_ESCAPE); + if (!escaped_etag) + return -ENOMEM; + } + + path = strjoin(image_root, "/", strempty(prefix), escaped_url, escaped_etag ? "." : "", + strempty(escaped_etag), strempty(suffix)); + if (!path) + return -ENOMEM; + + /* URLs might make the path longer than the maximum allowed length for a file name. + * When that happens, a URL hash is used instead. Paths returned by this function + * can be later used with tempfn_random() which adds 16 bytes to the resulting name. */ + if (strlen(path) >= HASH_URL_THRESHOLD_LENGTH) { + _cleanup_free_ char *hash = NULL; + int r; + + free(path); + + r = hash_url(url, &hash); + if (r < 0) + return r; + + path = strjoin(image_root, "/", strempty(prefix), hash, escaped_etag ? "." : "", + strempty(escaped_etag), strempty(suffix)); + if (!path) + return -ENOMEM; + } + + *ret = path; + return 0; +} + +int pull_make_auxiliary_job( + PullJob **ret, + const char *url, + int (*strip_suffixes)(const char *name, char **ret), + const char *suffix, + ImportVerify verify, + CurlGlue *glue, + PullJobOpenDisk on_open_disk, + PullJobFinished on_finished, + void *userdata) { + + _cleanup_free_ char *last_component = NULL, *ll = NULL, *auxiliary_url = NULL; + _cleanup_(pull_job_unrefp) PullJob *job = NULL; + const char *q; + int r; + + assert(ret); + assert(url); + assert(strip_suffixes); + assert(glue); + + r = import_url_last_component(url, &last_component); + if (r < 0) + return r; + + r = strip_suffixes(last_component, &ll); + if (r < 0) + return r; + + q = strjoina(ll, suffix); + + r = import_url_change_last_component(url, q, &auxiliary_url); + if (r < 0) + return r; + + r = pull_job_new(&job, auxiliary_url, glue, userdata); + if (r < 0) + return r; + + job->on_open_disk = on_open_disk; + job->on_finished = on_finished; + job->compressed_max = job->uncompressed_max = 1ULL * 1024ULL * 1024ULL; + job->calc_checksum = IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE); + + *ret = TAKE_PTR(job); + return 0; +} + +static bool is_checksum_file(const char *fn) { + /* Returns true if the specified filename refers to a checksum file we grok */ + + if (!fn) + return false; + + return streq(fn, "SHA256SUMS") || endswith(fn, ".sha256"); +} + +static bool is_signature_file(const char *fn) { + /* Returns true if the specified filename refers to a signature file we grok (reminder: + * suse-style .sha256 files are inline signed) */ + + if (!fn) + return false; + + return streq(fn, "SHA256SUMS.gpg") || endswith(fn, ".sha256"); +} + +int pull_make_verification_jobs( + PullJob **ret_checksum_job, + PullJob **ret_signature_job, + ImportVerify verify, + const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */ + const char *url, + CurlGlue *glue, + PullJobFinished on_finished, + void *userdata) { + + _cleanup_(pull_job_unrefp) PullJob *checksum_job = NULL, *signature_job = NULL; + _cleanup_free_ char *fn = NULL; + int r; + + assert(ret_checksum_job); + assert(ret_signature_job); + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); + assert(url); + assert(glue); + + /* If verification is turned off, or if the checksum to validate is already specified we don't need + * to download a checksum file or signature, hence shortcut things */ + if (verify == IMPORT_VERIFY_NO || checksum) { + *ret_checksum_job = *ret_signature_job = NULL; + return 0; + } + + r = import_url_last_component(url, &fn); + if (r < 0 && r != -EADDRNOTAVAIL) /* EADDRNOTAVAIL means there was no last component, which is OK for + * us, we'll just assume it's not a checksum/signature file */ + return r; + + /* Acquire the checksum file if verification or signature verification is requested and the main file + * to acquire isn't a checksum or signature file anyway */ + if (verify != IMPORT_VERIFY_NO && !is_checksum_file(fn) && !is_signature_file(fn)) { + _cleanup_free_ char *checksum_url = NULL; + const char *suffixed = NULL; + + /* Queue jobs for the checksum file for the image. */ + + if (fn) + suffixed = strjoina(fn, ".sha256"); /* Start with the suse-style checksum (if there's a base filename) */ + else + suffixed = "SHA256SUMS"; + + r = import_url_change_last_component(url, suffixed, &checksum_url); + if (r < 0) + return r; + + r = pull_job_new(&checksum_job, checksum_url, glue, userdata); + if (r < 0) + return r; + + checksum_job->on_finished = on_finished; + checksum_job->uncompressed_max = checksum_job->compressed_max = 1ULL * 1024ULL * 1024ULL; + checksum_job->on_not_found = pull_job_restart_with_sha256sum; /* if this fails, look for ubuntu-style checksum */ + } + + if (verify == IMPORT_VERIFY_SIGNATURE && !is_signature_file(fn)) { + _cleanup_free_ char *signature_url = NULL; + + /* Queue job for the SHA256SUMS.gpg file for the image. */ + r = import_url_change_last_component(url, "SHA256SUMS.gpg", &signature_url); + if (r < 0) + return r; + + r = pull_job_new(&signature_job, signature_url, glue, userdata); + if (r < 0) + return r; + + signature_job->on_finished = on_finished; + signature_job->uncompressed_max = signature_job->compressed_max = 1ULL * 1024ULL * 1024ULL; + } + + *ret_checksum_job = TAKE_PTR(checksum_job); + *ret_signature_job = TAKE_PTR(signature_job); + return 0; +} + +static int verify_one(PullJob *checksum_job, PullJob *job) { + _cleanup_free_ char *fn = NULL; + const char *line, *p; + int r; + + assert(checksum_job); + + if (!job) + return 0; + + assert(IN_SET(job->state, PULL_JOB_DONE, PULL_JOB_FAILED)); + + /* Don't verify the checksum if we didn't actually successfully download something new */ + if (job->state != PULL_JOB_DONE) + return 0; + if (job->error != 0) + return 0; + if (job->etag_exists) + return 0; + + assert(job->calc_checksum); + assert(job->checksum); + + r = import_url_last_component(job->url, &fn); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from URL '%s': %m", job->url); + + if (!filename_is_valid(fn)) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "Cannot verify checksum, could not determine server-side file name."); + + if (is_checksum_file(fn) || is_signature_file(fn)) /* We cannot verify checksum files or signature files with a checksum file */ + return log_error_errno(SYNTHETIC_ERRNO(ELOOP), + "Cannot verify checksum/signature files via themselves."); + + line = strjoina(job->checksum, " *", fn, "\n"); /* string for binary mode */ + p = memmem_safe(checksum_job->payload, + checksum_job->payload_size, + line, + strlen(line)); + if (!p) { + line = strjoina(job->checksum, " ", fn, "\n"); /* string for text mode */ + p = memmem_safe(checksum_job->payload, + checksum_job->payload_size, + line, + strlen(line)); + } + + /* Only counts if found at beginning of a line */ + if (!p || (p != (char*) checksum_job->payload && p[-1] != '\n')) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", fn); + + log_info("SHA256 checksum of %s is valid.", job->url); + return 1; +} + +static int verify_gpg( + const void *payload, size_t payload_size, + const void *signature, size_t signature_size) { + + _cleanup_close_pair_ int gpg_pipe[2] = { -1, -1 }; + char sig_file_path[] = "/tmp/sigXXXXXX", gpg_home[] = "/tmp/gpghomeXXXXXX"; + _cleanup_(sigkill_waitp) pid_t pid = 0; + bool gpg_home_created = false; + int r; + + assert(payload || payload_size == 0); + assert(signature || signature_size == 0); + + r = pipe2(gpg_pipe, O_CLOEXEC); + if (r < 0) + return log_error_errno(errno, "Failed to create pipe for gpg: %m"); + + if (signature_size > 0) { + _cleanup_close_ int sig_file = -1; + + sig_file = mkostemp(sig_file_path, O_RDWR); + if (sig_file < 0) + return log_error_errno(errno, "Failed to create temporary file: %m"); + + r = loop_write(sig_file, signature, signature_size, false); + if (r < 0) { + log_error_errno(r, "Failed to write to temporary file: %m"); + goto finish; + } + } + + if (!mkdtemp(gpg_home)) { + r = log_error_errno(errno, "Failed to create temporary home for gpg: %m"); + goto finish; + } + + gpg_home_created = true; + + r = safe_fork("(gpg)", FORK_RESET_SIGNALS|FORK_DEATHSIG|FORK_LOG, &pid); + if (r < 0) + return r; + if (r == 0) { + const char *cmd[] = { + "gpg", + "--no-options", + "--no-default-keyring", + "--no-auto-key-locate", + "--no-auto-check-trustdb", + "--batch", + "--trust-model=always", + NULL, /* --homedir= */ + NULL, /* --keyring= */ + NULL, /* --verify */ + NULL, /* signature file */ + NULL, /* dash */ + NULL /* trailing NULL */ + }; + size_t k = ELEMENTSOF(cmd) - 6; + + /* Child */ + + gpg_pipe[1] = safe_close(gpg_pipe[1]); + + r = rearrange_stdio(TAKE_FD(gpg_pipe[0]), -1, STDERR_FILENO); + if (r < 0) { + log_error_errno(r, "Failed to rearrange stdin/stdout: %m"); + _exit(EXIT_FAILURE); + } + + (void) rlimit_nofile_safe(); + + cmd[k++] = strjoina("--homedir=", gpg_home); + + /* We add the user keyring only to the command line arguments, if it's around since gpg fails + * otherwise. */ + if (access(USER_KEYRING_PATH, F_OK) >= 0) + cmd[k++] = "--keyring=" USER_KEYRING_PATH; + else + cmd[k++] = "--keyring=" VENDOR_KEYRING_PATH; + + cmd[k++] = "--verify"; + if (signature) { + cmd[k++] = sig_file_path; + cmd[k++] = "-"; + cmd[k++] = NULL; + } + + execvp("gpg2", (char * const *) cmd); + execvp("gpg", (char * const *) cmd); + log_error_errno(errno, "Failed to execute gpg: %m"); + _exit(EXIT_FAILURE); + } + + gpg_pipe[0] = safe_close(gpg_pipe[0]); + + r = loop_write(gpg_pipe[1], payload, payload_size, false); + if (r < 0) { + log_error_errno(r, "Failed to write to pipe: %m"); + goto finish; + } + + gpg_pipe[1] = safe_close(gpg_pipe[1]); + + r = wait_for_terminate_and_check("gpg", TAKE_PID(pid), WAIT_LOG_ABNORMAL); + if (r < 0) + goto finish; + if (r != EXIT_SUCCESS) + r = log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "DOWNLOAD INVALID: Signature verification failed."); + else { + log_info("Signature verification succeeded."); + r = 0; + } + +finish: + if (signature_size > 0) + (void) unlink(sig_file_path); + + if (gpg_home_created) + (void) rm_rf(gpg_home, REMOVE_ROOT|REMOVE_PHYSICAL); + + return r; +} + +int pull_verify(ImportVerify verify, + const char *checksum, /* Verify with literal checksum */ + PullJob *main_job, + PullJob *checksum_job, + PullJob *signature_job, + PullJob *settings_job, + PullJob *roothash_job, + PullJob *roothash_signature_job, + PullJob *verity_job) { + + _cleanup_free_ char *fn = NULL; + VerificationStyle style; + PullJob *verify_job; + int r; + + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); + assert(main_job); + assert(main_job->state == PULL_JOB_DONE); + + if (verify == IMPORT_VERIFY_NO) /* verification turned off */ + return 0; + + if (checksum) { + /* Verification by literal checksum */ + assert(!checksum_job); + assert(!signature_job); + assert(!settings_job); + assert(!roothash_job); + assert(!roothash_signature_job); + assert(!verity_job); + + assert(main_job->calc_checksum); + assert(main_job->checksum); + + if (!strcaseeq(checksum, main_job->checksum)) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", + main_job->url); + + return 0; + } + + r = import_url_last_component(main_job->url, &fn); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from URL '%s': %m", main_job->url); + + if (is_signature_file(fn)) + return log_error_errno(SYNTHETIC_ERRNO(ELOOP), + "Main download is a signature file, can't verify it."); + + if (is_checksum_file(fn)) { + log_debug("Main download is a checksum file, can't validate its checksum with itself, skipping."); + verify_job = main_job; + } else { + PullJob *j; + assert(main_job->calc_checksum); + assert(main_job->checksum); + assert(checksum_job); + assert(checksum_job->state == PULL_JOB_DONE); + + if (!checksum_job->payload || checksum_job->payload_size <= 0) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "Checksum is empty, cannot verify."); + + FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) { + r = verify_one(checksum_job, j); + if (r < 0) + return r; + } + + verify_job = checksum_job; + } + + if (verify != IMPORT_VERIFY_SIGNATURE) + return 0; + + assert(verify_job); + + r = verification_style_from_url(verify_job->url, &style); + if (r < 0) + return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", verify_job->url); + + if (style == VERIFICATION_PER_DIRECTORY) { + assert(signature_job); + assert(signature_job->state == PULL_JOB_DONE); + + if (!signature_job->payload || signature_job->payload_size <= 0) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "Signature is empty, cannot verify."); + + return verify_gpg(verify_job->payload, verify_job->payload_size, signature_job->payload, signature_job->payload_size); + } else + return verify_gpg(verify_job->payload, verify_job->payload_size, NULL, 0); +} + +int verification_style_from_url(const char *url, VerificationStyle *ret) { + _cleanup_free_ char *last = NULL; + int r; + + assert(url); + assert(ret); + + /* Determines which kind of verification style is appropriate for this url */ + + r = import_url_last_component(url, &last); + if (r < 0) + return r; + + if (streq(last, "SHA256SUMS")) { + *ret = VERIFICATION_PER_DIRECTORY; + return 0; + } + + if (endswith(last, ".sha256")) { + *ret = VERIFICATION_PER_FILE; + return 0; + } + + return -EINVAL; +} + +int pull_job_restart_with_sha256sum(PullJob *j, char **ret) { + VerificationStyle style; + int r; + + assert(j); + + /* Generic implementation of a PullJobNotFound handler, that restarts the job requesting SHA256SUMS */ + + r = verification_style_from_url(j->url, &style); + if (r < 0) + return log_error_errno(r, "Failed to determine verification style of URL '%s': %m", j->url); + + if (style == VERIFICATION_PER_DIRECTORY) /* Nothing to do anymore */ + return 0; + + assert(style == VERIFICATION_PER_FILE); /* This must have been .sha256 style URL before */ + + log_debug("Got 404 for %s, now trying to get SHA256SUMS instead.", j->url); + + r = import_url_change_last_component(j->url, "SHA256SUMS", ret); + if (r < 0) + return log_error_errno(r, "Failed to replace SHA256SUMS suffix: %m"); + + return 1; +} + +bool pull_validate_local(const char *name, PullFlags flags) { + + if (FLAGS_SET(flags, PULL_DIRECT)) + return path_is_valid(name); + + return hostname_is_valid(name, 0); +} + +int pull_url_needs_checksum(const char *url) { + _cleanup_free_ char *fn = NULL; + int r; + + /* Returns true if we need to validate this resource via a hash value. This returns true for all + * files — except for gpg signature files and SHA256SUMS files and the like, which are validated with + * a validation tool like gpg. */ + + r = import_url_last_component(url, &fn); + if (r == -EADDRNOTAVAIL) /* no last component? then let's assume it's not a signature/checksum file */ + return false; + if (r < 0) + return r; + + return !is_checksum_file(fn) && !is_signature_file(fn); +} diff --git a/src/import/pull-common.h b/src/import/pull-common.h new file mode 100644 index 0000000..475613a --- /dev/null +++ b/src/import/pull-common.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include <stdbool.h> + +#include "import-util.h" +#include "pull-job.h" + +typedef enum PullFlags { + PULL_FORCE = 1 << 0, /* replace existing image */ + PULL_READ_ONLY = 1 << 1, /* make generated image read-only */ + PULL_SETTINGS = 1 << 2, /* download .nspawn settings file */ + PULL_ROOTHASH = 1 << 3, /* only for raw: download .roothash file for verity */ + PULL_ROOTHASH_SIGNATURE = 1 << 4, /* only for raw: download .roothash.p7s file for verity */ + PULL_VERITY = 1 << 5, /* only for raw: download .verity file for verity */ + PULL_BTRFS_SUBVOL = 1 << 6, /* tar: preferably create images as btrfs subvols */ + PULL_BTRFS_QUOTA = 1 << 7, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ + PULL_CONVERT_QCOW2 = 1 << 8, /* raw: if we detect a qcow2 image, unpack it */ + PULL_DIRECT = 1 << 9, /* download without rename games */ + PULL_SYNC = 1 << 10, /* fsync() right before we are done */ + + /* The supported flags for the tar and the raw pulling */ + PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_BTRFS_SUBVOL|PULL_BTRFS_QUOTA|PULL_DIRECT|PULL_SYNC, + PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY|PULL_CONVERT_QCOW2|PULL_DIRECT|PULL_SYNC, +} PullFlags; + +int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags); + +int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret); + +int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata); +int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); + +int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job); + +typedef enum VerificationStyle { + VERIFICATION_PER_FILE, /* SuSE-style ".sha256" files with inline gpg signature */ + VERIFICATION_PER_DIRECTORY, /* Ubuntu-style SHA256SUM files with detached SHA256SUM.gpg signatures */ + _VERIFICATION_STYLE_MAX, + _VERIFICATION_STYLE_INVALID = -EINVAL, +} VerificationStyle; + +int verification_style_from_url(const char *url, VerificationStyle *style); + +int pull_job_restart_with_sha256sum(PullJob *job, char **ret); + +bool pull_validate_local(const char *name, PullFlags flags); + +int pull_url_needs_checksum(const char *url); diff --git a/src/import/pull-job.c b/src/import/pull-job.c new file mode 100644 index 0000000..d4d07a0 --- /dev/null +++ b/src/import/pull-job.c @@ -0,0 +1,784 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/xattr.h> + +#include "alloc-util.h" +#include "fd-util.h" +#include "format-util.h" +#include "gcrypt-util.h" +#include "hexdecoct.h" +#include "import-util.h" +#include "io-util.h" +#include "machine-pool.h" +#include "parse-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "string-util.h" +#include "strv.h" +#include "sync-util.h" +#include "xattr-util.h" + +void pull_job_close_disk_fd(PullJob *j) { + if (!j) + return; + + if (j->close_disk_fd) + safe_close(j->disk_fd); + + j->disk_fd = -1; +} + +PullJob* pull_job_unref(PullJob *j) { + if (!j) + return NULL; + + pull_job_close_disk_fd(j); + + curl_glue_remove_and_free(j->glue, j->curl); + curl_slist_free_all(j->request_header); + + import_compress_free(&j->compress); + + if (j->checksum_ctx) +#if PREFER_OPENSSL + EVP_MD_CTX_free(j->checksum_ctx); +#else + gcry_md_close(j->checksum_ctx); +#endif + + free(j->url); + free(j->etag); + strv_free(j->old_etags); + free(j->payload); + free(j->checksum); + + return mfree(j); +} + +static void pull_job_finish(PullJob *j, int ret) { + assert(j); + + if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) + return; + + if (ret == 0) { + j->state = PULL_JOB_DONE; + j->progress_percent = 100; + log_info("Download of %s complete.", j->url); + } else { + j->state = PULL_JOB_FAILED; + j->error = ret; + } + + if (j->on_finished) + j->on_finished(j); +} + +static int pull_job_restart(PullJob *j, const char *new_url) { + int r; + + assert(j); + assert(new_url); + + r = free_and_strdup(&j->url, new_url); + if (r < 0) + return r; + + j->state = PULL_JOB_INIT; + j->error = 0; + j->payload = mfree(j->payload); + j->payload_size = 0; + j->written_compressed = 0; + j->written_uncompressed = 0; + j->content_length = UINT64_MAX; + j->etag = mfree(j->etag); + j->etag_exists = false; + j->mtime = 0; + j->checksum = mfree(j->checksum); + + curl_glue_remove_and_free(j->glue, j->curl); + j->curl = NULL; + + curl_slist_free_all(j->request_header); + j->request_header = NULL; + + import_compress_free(&j->compress); + + if (j->checksum_ctx) { +#if PREFER_OPENSSL + EVP_MD_CTX_free(j->checksum_ctx); +#else + gcry_md_close(j->checksum_ctx); +#endif + j->checksum_ctx = NULL; + } + + r = pull_job_begin(j); + if (r < 0) + return r; + + return 0; +} + +void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { + PullJob *j = NULL; + char *scheme = NULL; + CURLcode code; + int r; + + if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK) + return; + + if (!j || IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) + return; + + if (result != CURLE_OK) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Transfer failed: %s", curl_easy_strerror(result)); + goto finish; + } + + code = curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme); + if (code != CURLE_OK || !scheme) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve URL scheme."); + goto finish; + } + + if (STRCASE_IN_SET(scheme, "HTTP", "HTTPS")) { + long status; + + code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); + if (code != CURLE_OK) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code)); + goto finish; + } + + if (status == 304) { + log_info("Image already downloaded. Skipping download."); + j->etag_exists = true; + r = 0; + goto finish; + } else if (status >= 300) { + + if (status == 404 && j->on_not_found) { + _cleanup_free_ char *new_url = NULL; + + /* This resource wasn't found, but the implementor wants to maybe let us know a new URL, query for it. */ + r = j->on_not_found(j, &new_url); + if (r < 0) + goto finish; + + if (r > 0) { /* A new url to use */ + assert(new_url); + + r = pull_job_restart(j, new_url); + if (r < 0) + goto finish; + + code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status); + if (code != CURLE_OK) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code)); + goto finish; + } + + if (status == 0) + return; + } + } + + r = log_error_errno( + status == 404 ? SYNTHETIC_ERRNO(ENOMEDIUM) : SYNTHETIC_ERRNO(EIO), /* Make the most common error recognizable */ + "HTTP request to %s failed with code %li.", j->url, status); + goto finish; + } else if (status < 200) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "HTTP request to %s finished with unexpected code %li.", j->url, status); + goto finish; + } + } + + if (j->state != PULL_JOB_RUNNING) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination."); + goto finish; + } + + if (j->content_length != UINT64_MAX && + j->content_length != j->written_compressed) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Download truncated."); + goto finish; + } + + if (j->checksum_ctx) { + unsigned checksum_len; +#if PREFER_OPENSSL + uint8_t k[EVP_MAX_MD_SIZE]; + + r = EVP_DigestFinal_ex(j->checksum_ctx, k, &checksum_len); + if (r == 0) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum."); + goto finish; + } + assert(checksum_len <= sizeof k); +#else + const uint8_t *k; + + k = gcry_md_read(j->checksum_ctx, GCRY_MD_SHA256); + if (!k) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum."); + goto finish; + } + + checksum_len = gcry_md_get_algo_dlen(GCRY_MD_SHA256); +#endif + + j->checksum = hexmem(k, checksum_len); + if (!j->checksum) { + r = log_oom(); + goto finish; + } + + log_debug("SHA256 of %s is %s.", j->url, j->checksum); + } + + /* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no + * offset is specified, which indicates we only own the file partially) */ + + if (j->disk_fd >= 0) { + + if (S_ISREG(j->disk_stat.st_mode)) { + + if (j->offset == UINT64_MAX) { + + if (j->written_compressed > 0) { + /* Make sure the file size is right, in case the file was sparse and we just seeked + * for the last part */ + if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) { + r = log_error_errno(errno, "Failed to truncate file: %m"); + goto finish; + } + } + + if (j->etag) + (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0); + if (j->url) + (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0); + + if (j->mtime != 0) { + struct timespec ut; + + timespec_store(&ut, j->mtime); + + if (futimens(j->disk_fd, (struct timespec[]) { ut, ut }) < 0) + log_debug_errno(errno, "Failed to adjust atime/mtime of created image, ignoring: %m"); + + r = fd_setcrtime(j->disk_fd, j->mtime); + if (r < 0) + log_debug_errno(r, "Failed to adjust crtime of created image, ignoring: %m"); + } + } + + if (j->sync) { + r = fsync_full(j->disk_fd); + if (r < 0) { + log_error_errno(r, "Failed to synchronize file to disk: %m"); + goto finish; + } + } + + } else if (S_ISBLK(j->disk_stat.st_mode) && j->sync) { + + if (fsync(j->disk_fd) < 0) { + r = log_error_errno(errno, "Failed to synchronize block device: %m"); + goto finish; + } + } + } + + log_info("Acquired %s.", FORMAT_BYTES(j->written_uncompressed)); + + r = 0; + +finish: + pull_job_finish(j, r); +} + +static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) { + PullJob *j = ASSERT_PTR(userdata); + bool too_much = false; + int r; + + assert(p); + assert(sz > 0); + + if (j->written_uncompressed > UINT64_MAX - sz) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); + + if (j->written_uncompressed >= j->uncompressed_max) { + too_much = true; + goto finish; + } + + if (j->written_uncompressed + sz > j->uncompressed_max) { + too_much = true; + sz = j->uncompressed_max - j->written_uncompressed; /* since we have the data in memory + * already, we might as well write it to + * disk to the max */ + } + + if (j->disk_fd >= 0) { + + if (S_ISREG(j->disk_stat.st_mode) && j->offset == UINT64_MAX) { + ssize_t n; + + n = sparse_write(j->disk_fd, p, sz, 64); + if (n < 0) + return log_error_errno((int) n, "Failed to write file: %m"); + if ((size_t) n < sz) + return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); + } else { + r = loop_write(j->disk_fd, p, sz, false); + if (r < 0) + return log_error_errno(r, "Failed to write file: %m"); + } + } + + if (j->disk_fd < 0 || j->force_memory) { + if (!GREEDY_REALLOC(j->payload, j->payload_size + sz)) + return log_oom(); + + memcpy(j->payload + j->payload_size, p, sz); + j->payload_size += sz; + } + + j->written_uncompressed += sz; + +finish: + if (too_much) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); + + return 0; +} + +static int pull_job_write_compressed(PullJob *j, void *p, size_t sz) { + int r; + + assert(j); + assert(p); + + if (sz <= 0) + return 0; + + if (j->written_compressed + sz < j->written_compressed) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); + + if (j->written_compressed + sz > j->compressed_max) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); + + if (j->content_length != UINT64_MAX && + j->written_compressed + sz > j->content_length) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), + "Content length incorrect."); + + if (j->checksum_ctx) { +#if PREFER_OPENSSL + r = EVP_DigestUpdate(j->checksum_ctx, p, sz); + if (r == 0) + return log_error_errno(SYNTHETIC_ERRNO(EIO), + "Could not hash chunk."); +#else + gcry_md_write(j->checksum_ctx, p, sz); +#endif + } + + r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j); + if (r < 0) + return r; + + j->written_compressed += sz; + + return 0; +} + +static int pull_job_open_disk(PullJob *j) { + int r; + + assert(j); + + if (j->on_open_disk) { + r = j->on_open_disk(j); + if (r < 0) + return r; + } + + if (j->disk_fd >= 0) { + if (fstat(j->disk_fd, &j->disk_stat) < 0) + return log_error_errno(errno, "Failed to stat disk file: %m"); + + if (j->offset != UINT64_MAX) { + if (lseek(j->disk_fd, j->offset, SEEK_SET) == (off_t) -1) + return log_error_errno(errno, "Failed to seek on file descriptor: %m"); + } + } + + if (j->calc_checksum) { +#if PREFER_OPENSSL + j->checksum_ctx = EVP_MD_CTX_new(); + if (!j->checksum_ctx) + return log_oom(); + + r = EVP_DigestInit_ex(j->checksum_ctx, EVP_sha256(), NULL); + if (r == 0) + return log_error_errno(SYNTHETIC_ERRNO(EIO), + "Failed to initialize hash context."); +#else + initialize_libgcrypt(false); + + if (gcry_md_open(&j->checksum_ctx, GCRY_MD_SHA256, 0) != 0) + return log_error_errno(SYNTHETIC_ERRNO(EIO), + "Failed to initialize hash context."); +#endif + } + + return 0; +} + +static int pull_job_detect_compression(PullJob *j) { + _cleanup_free_ uint8_t *stub = NULL; + size_t stub_size; + + int r; + + assert(j); + + r = import_uncompress_detect(&j->compress, j->payload, j->payload_size); + if (r < 0) + return log_error_errno(r, "Failed to initialize compressor: %m"); + if (r == 0) + return 0; + + log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type)); + + r = pull_job_open_disk(j); + if (r < 0) + return r; + + /* Now, take the payload we read so far, and decompress it */ + stub = j->payload; + stub_size = j->payload_size; + + j->payload = NULL; + j->payload_size = 0; + + j->state = PULL_JOB_RUNNING; + + r = pull_job_write_compressed(j, stub, stub_size); + if (r < 0) + return r; + + return 0; +} + +static size_t pull_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) { + PullJob *j = ASSERT_PTR(userdata); + size_t sz = size * nmemb; + int r; + + assert(contents); + + switch (j->state) { + + case PULL_JOB_ANALYZING: + /* Let's first check what it actually is */ + + if (!GREEDY_REALLOC(j->payload, j->payload_size + sz)) { + r = log_oom(); + goto fail; + } + + memcpy(j->payload + j->payload_size, contents, sz); + j->payload_size += sz; + + r = pull_job_detect_compression(j); + if (r < 0) + goto fail; + + break; + + case PULL_JOB_RUNNING: + + r = pull_job_write_compressed(j, contents, sz); + if (r < 0) + goto fail; + + break; + + case PULL_JOB_DONE: + case PULL_JOB_FAILED: + r = -ESTALE; + goto fail; + + default: + assert_not_reached(); + } + + return sz; + +fail: + pull_job_finish(j, r); + return 0; +} + +static int http_status_ok(CURLcode status) { + /* Consider all HTTP status code in the 2xx range as OK */ + return status >= 200 && status <= 299; +} + +static int http_status_etag_exists(CURLcode status) { + /* This one is special, it's triggered by our etag mgmt logic */ + return status == 304; +} + +static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) { + _cleanup_free_ char *length = NULL, *last_modified = NULL, *etag = NULL; + size_t sz = size * nmemb; + PullJob *j = ASSERT_PTR(userdata); + CURLcode code; + long status; + int r; + + assert(contents); + + if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) { + r = -ESTALE; + goto fail; + } + + assert(j->state == PULL_JOB_ANALYZING); + + code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status); + if (code != CURLE_OK) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code)); + goto fail; + } + + if (http_status_ok(status) || http_status_etag_exists(status)) { + /* Check Etag on OK and etag exists responses. */ + + r = curl_header_strdup(contents, sz, "ETag:", &etag); + if (r < 0) { + log_oom(); + goto fail; + } + if (r > 0) { + free_and_replace(j->etag, etag); + + if (strv_contains(j->old_etags, j->etag)) { + log_info("Image already downloaded. Skipping download. (%s)", j->etag); + j->etag_exists = true; + pull_job_finish(j, 0); + return sz; + } + + return sz; + } + } + + if (!http_status_ok(status)) /* Let's ignore the rest here, these requests are probably redirects and + * stuff where the headers aren't interesting to us */ + return sz; + + r = curl_header_strdup(contents, sz, "Content-Length:", &length); + if (r < 0) { + log_oom(); + goto fail; + } + if (r > 0) { + (void) safe_atou64(length, &j->content_length); + + if (j->content_length != UINT64_MAX) { + if (j->content_length > j->compressed_max) { + r = log_error_errno(SYNTHETIC_ERRNO(EFBIG), "Content too large."); + goto fail; + } + + log_info("Downloading %s for %s.", FORMAT_BYTES(j->content_length), j->url); + } + + return sz; + } + + r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified); + if (r < 0) { + log_oom(); + goto fail; + } + if (r > 0) { + (void) curl_parse_http_time(last_modified, &j->mtime); + return sz; + } + + if (j->on_header) { + r = j->on_header(j, contents, sz); + if (r < 0) + goto fail; + } + + return sz; + +fail: + pull_job_finish(j, r); + return 0; +} + +static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { + PullJob *j = ASSERT_PTR(userdata); + unsigned percent; + usec_t n; + + if (dltotal <= 0) + return 0; + + percent = ((100 * dlnow) / dltotal); + n = now(CLOCK_MONOTONIC); + + if (n > j->last_status_usec + USEC_PER_SEC && + percent != j->progress_percent && + dlnow < dltotal) { + + if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) { + usec_t left, done; + + done = n - j->start_usec; + left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done; + + log_info("Got %u%% of %s. %s left at %s/s.", + percent, + j->url, + FORMAT_TIMESPAN(left, USEC_PER_SEC), + FORMAT_BYTES((uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC)))); + } else + log_info("Got %u%% of %s.", percent, j->url); + + j->progress_percent = percent; + j->last_status_usec = n; + + if (j->on_progress) + j->on_progress(j); + } + + return 0; +} + +int pull_job_new( + PullJob **ret, + const char *url, + CurlGlue *glue, + void *userdata) { + + _cleanup_(pull_job_unrefp) PullJob *j = NULL; + _cleanup_free_ char *u = NULL; + + assert(url); + assert(glue); + assert(ret); + + u = strdup(url); + if (!u) + return -ENOMEM; + + j = new(PullJob, 1); + if (!j) + return -ENOMEM; + + *j = (PullJob) { + .state = PULL_JOB_INIT, + .disk_fd = -1, + .close_disk_fd = true, + .userdata = userdata, + .glue = glue, + .content_length = UINT64_MAX, + .start_usec = now(CLOCK_MONOTONIC), + .compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */ + .uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */ + .url = TAKE_PTR(u), + .offset = UINT64_MAX, + .sync = true, + }; + + *ret = TAKE_PTR(j); + + return 0; +} + +int pull_job_begin(PullJob *j) { + int r; + + assert(j); + + if (j->state != PULL_JOB_INIT) + return -EBUSY; + + r = curl_glue_make(&j->curl, j->url, j); + if (r < 0) + return r; + + if (!strv_isempty(j->old_etags)) { + _cleanup_free_ char *cc = NULL, *hdr = NULL; + + cc = strv_join(j->old_etags, ", "); + if (!cc) + return -ENOMEM; + + hdr = strjoin("If-None-Match: ", cc); + if (!hdr) + return -ENOMEM; + + if (!j->request_header) { + j->request_header = curl_slist_new(hdr, NULL); + if (!j->request_header) + return -ENOMEM; + } else { + struct curl_slist *l; + + l = curl_slist_append(j->request_header, hdr); + if (!l) + return -ENOMEM; + + j->request_header = l; + } + } + + if (j->request_header) { + if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK) + return -EIO; + } + + if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK) + return -EIO; + + if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK) + return -EIO; + + r = curl_glue_add(j->glue, j->curl); + if (r < 0) + return r; + + j->state = PULL_JOB_ANALYZING; + + return 0; +} diff --git a/src/import/pull-job.h b/src/import/pull-job.h new file mode 100644 index 0000000..7a98b0f --- /dev/null +++ b/src/import/pull-job.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include <sys/stat.h> + +#include "curl-util.h" +#include "import-compress.h" +#include "macro.h" +#include "openssl-util.h" +#include "pull-common.h" + +typedef struct PullJob PullJob; + +typedef void (*PullJobFinished)(PullJob *job); +typedef int (*PullJobOpenDisk)(PullJob *job); +typedef int (*PullJobHeader)(PullJob *job, const char *header, size_t sz); +typedef void (*PullJobProgress)(PullJob *job); +typedef int (*PullJobNotFound)(PullJob *job, char **ret_new_url); + +typedef enum PullJobState { + PULL_JOB_INIT, + PULL_JOB_ANALYZING, /* Still reading into ->payload, to figure out what we have */ + PULL_JOB_RUNNING, /* Writing to destination */ + PULL_JOB_DONE, + PULL_JOB_FAILED, + _PULL_JOB_STATE_MAX, + _PULL_JOB_STATE_INVALID = -EINVAL, +} PullJobState; + +#define PULL_JOB_IS_COMPLETE(j) (IN_SET((j)->state, PULL_JOB_DONE, PULL_JOB_FAILED)) + +struct PullJob { + PullJobState state; + int error; + + char *url; + + void *userdata; + PullJobFinished on_finished; + PullJobOpenDisk on_open_disk; + PullJobHeader on_header; + PullJobProgress on_progress; + PullJobNotFound on_not_found; + + CurlGlue *glue; + CURL *curl; + struct curl_slist *request_header; + + char *etag; + char **old_etags; + bool etag_exists; + + uint64_t content_length; + uint64_t written_compressed; + uint64_t written_uncompressed; + uint64_t offset; + + uint64_t uncompressed_max; + uint64_t compressed_max; + + uint8_t *payload; + size_t payload_size; + + int disk_fd; + bool close_disk_fd; + struct stat disk_stat; + + usec_t mtime; + + ImportCompress compress; + + unsigned progress_percent; + usec_t start_usec; + usec_t last_status_usec; + + bool calc_checksum; + hash_context_t checksum_ctx; + + char *checksum; + bool sync; + bool force_memory; +}; + +int pull_job_new(PullJob **job, const char *url, CurlGlue *glue, void *userdata); +PullJob* pull_job_unref(PullJob *job); + +int pull_job_begin(PullJob *j); + +void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result); + +void pull_job_close_disk_fd(PullJob *j); + +DEFINE_TRIVIAL_CLEANUP_FUNC(PullJob*, pull_job_unref); diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c new file mode 100644 index 0000000..f4f869d --- /dev/null +++ b/src/import/pull-raw.c @@ -0,0 +1,985 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <curl/curl.h> +#include <linux/fs.h> +#include <sys/xattr.h> + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "curl-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-util.h" +#include "install-file.h" +#include "macro.h" +#include "mkdir-label.h" +#include "path-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "pull-raw.h" +#include "qcow2-util.h" +#include "rm-rf.h" +#include "string-util.h" +#include "strv.h" +#include "tmpfile-util.h" +#include "utf8.h" +#include "util.h" +#include "web-util.h" + +typedef enum RawProgress { + RAW_DOWNLOADING, + RAW_VERIFYING, + RAW_UNPACKING, + RAW_FINALIZING, + RAW_COPYING, +} RawProgress; + +struct RawPull { + sd_event *event; + CurlGlue *glue; + + PullFlags flags; + ImportVerify verify; + char *image_root; + + uint64_t offset; + + PullJob *raw_job; + PullJob *checksum_job; + PullJob *signature_job; + PullJob *settings_job; + PullJob *roothash_job; + PullJob *roothash_signature_job; + PullJob *verity_job; + + RawPullFinished on_finished; + void *userdata; + + char *local; /* In PULL_DIRECT mode the path we are supposed to place things in, otherwise the + * machine name of the final copy we make */ + + char *final_path; + char *temp_path; + + char *settings_path; + char *settings_temp_path; + + char *roothash_path; + char *roothash_temp_path; + + char *roothash_signature_path; + char *roothash_signature_temp_path; + + char *verity_path; + char *verity_temp_path; + + char *checksum; +}; + +RawPull* raw_pull_unref(RawPull *i) { + if (!i) + return NULL; + + pull_job_unref(i->raw_job); + pull_job_unref(i->checksum_job); + pull_job_unref(i->signature_job); + pull_job_unref(i->settings_job); + pull_job_unref(i->roothash_job); + pull_job_unref(i->roothash_signature_job); + pull_job_unref(i->verity_job); + + curl_glue_unref(i->glue); + sd_event_unref(i->event); + + unlink_and_free(i->temp_path); + unlink_and_free(i->settings_temp_path); + unlink_and_free(i->roothash_temp_path); + unlink_and_free(i->roothash_signature_temp_path); + unlink_and_free(i->verity_temp_path); + + free(i->final_path); + free(i->settings_path); + free(i->roothash_path); + free(i->roothash_signature_path); + free(i->verity_path); + free(i->image_root); + free(i->local); + free(i->checksum); + + return mfree(i); +} + +int raw_pull_new( + RawPull **ret, + sd_event *event, + const char *image_root, + RawPullFinished on_finished, + void *userdata) { + + _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL; + _cleanup_(sd_event_unrefp) sd_event *e = NULL; + _cleanup_(raw_pull_unrefp) RawPull *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + if (event) + e = sd_event_ref(event); + else { + r = sd_event_default(&e); + if (r < 0) + return r; + } + + r = curl_glue_new(&g, e); + if (r < 0) + return r; + + i = new(RawPull, 1); + if (!i) + return -ENOMEM; + + *i = (RawPull) { + .on_finished = on_finished, + .userdata = userdata, + .image_root = TAKE_PTR(root), + .event = TAKE_PTR(e), + .glue = TAKE_PTR(g), + .offset = UINT64_MAX, + }; + + i->glue->on_finished = pull_job_curl_on_finished; + i->glue->userdata = i; + + *ret = TAKE_PTR(i); + + return 0; +} + +static void raw_pull_report_progress(RawPull *i, RawProgress p) { + unsigned percent; + + assert(i); + + switch (p) { + + case RAW_DOWNLOADING: { + unsigned remain = 80; + + percent = 0; + + if (i->checksum_job) { + percent += i->checksum_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->signature_job) { + percent += i->signature_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->settings_job) { + percent += i->settings_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->roothash_job) { + percent += i->roothash_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->roothash_signature_job) { + percent += i->roothash_signature_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->verity_job) { + percent += i->verity_job->progress_percent * 10 / 100; + remain -= 10; + } + + if (i->raw_job) + percent += i->raw_job->progress_percent * remain / 100; + break; + } + + case RAW_VERIFYING: + percent = 80; + break; + + case RAW_UNPACKING: + percent = 85; + break; + + case RAW_FINALIZING: + percent = 90; + break; + + case RAW_COPYING: + percent = 95; + break; + + default: + assert_not_reached(); + } + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_debug("Combined progress %u%%", percent); +} + +static int raw_pull_maybe_convert_qcow2(RawPull *i) { + _cleanup_(unlink_and_freep) char *t = NULL; + _cleanup_close_ int converted_fd = -1; + _cleanup_free_ char *f = NULL; + int r; + + assert(i); + assert(i->raw_job); + assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + + if (!FLAGS_SET(i->flags, PULL_CONVERT_QCOW2)) + return 0; + + assert(i->final_path); + assert(i->raw_job->close_disk_fd); + + r = qcow2_detect(i->raw_job->disk_fd); + if (r < 0) + return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); + if (r == 0) + return 0; + + /* This is a QCOW2 image, let's convert it */ + r = tempfn_random(i->final_path, NULL, &f); + if (r < 0) + return log_oom(); + + converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (converted_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", f); + + t = TAKE_PTR(f); + + (void) import_set_nocow_and_log(converted_fd, t); + + log_info("Unpacking QCOW2 file."); + + r = qcow2_convert(i->raw_job->disk_fd, converted_fd); + if (r < 0) + return log_error_errno(r, "Failed to convert qcow2 image: %m"); + + unlink_and_free(i->temp_path); + i->temp_path = TAKE_PTR(t); + close_and_replace(i->raw_job->disk_fd, converted_fd); + + return 1; +} + +static int raw_pull_determine_path( + RawPull *i, + const char *suffix, + char **field /* input + output (!) */) { + int r; + + assert(i); + assert(field); + + if (*field) + return 0; + + assert(i->raw_job); + + r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", suffix, field); + if (r < 0) + return log_oom(); + + return 1; +} + +static int raw_pull_copy_auxiliary_file( + RawPull *i, + const char *suffix, + char **path /* input + output (!) */) { + + const char *local; + int r; + + assert(i); + assert(suffix); + assert(path); + + r = raw_pull_determine_path(i, suffix, path); + if (r < 0) + return r; + + local = strjoina(i->image_root, "/", i->local, suffix); + + r = copy_file_atomic( + *path, + local, + 0644, + 0, 0, + COPY_REFLINK | + (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0)); + if (r == -EEXIST) + log_warning_errno(r, "File %s already exists, not replacing.", local); + else if (r == -ENOENT) + log_debug_errno(r, "Skipping creation of auxiliary file, since none was found."); + else if (r < 0) + log_warning_errno(r, "Failed to copy file %s, ignoring: %m", local); + else + log_info("Created new file %s.", local); + + return 0; +} + +static int raw_pull_make_local_copy(RawPull *i) { + _cleanup_(unlink_and_freep) char *tp = NULL; + _cleanup_free_ char *f = NULL; + _cleanup_close_ int dfd = -1; + const char *p; + int r; + + assert(i); + assert(i->raw_job); + assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + + if (!i->local) + return 0; + + if (i->raw_job->etag_exists) { + /* We have downloaded this one previously, reopen it */ + + assert(i->raw_job->disk_fd < 0); + + i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC); + if (i->raw_job->disk_fd < 0) + return log_error_errno(errno, "Failed to open vendor image: %m"); + } else { + /* We freshly downloaded the image, use it */ + + assert(i->raw_job->disk_fd >= 0); + assert(i->offset == UINT64_MAX); + + if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1) + return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m"); + } + + p = strjoina(i->image_root, "/", i->local, ".raw"); + + r = tempfn_random(p, NULL, &f); + if (r < 0) + return log_oom(); + + dfd = open(f, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (dfd < 0) + return log_error_errno(errno, "Failed to create writable copy of image: %m"); + + tp = TAKE_PTR(f); + + /* Turn off COW writing. This should greatly improve performance on COW file systems like btrfs, + * since it reduces fragmentation caused by not allowing in-place writes. */ + (void) import_set_nocow_and_log(dfd, tp); + + r = copy_bytes(i->raw_job->disk_fd, dfd, UINT64_MAX, COPY_REFLINK); + if (r < 0) + return log_error_errno(r, "Failed to make writable copy of image: %m"); + + (void) copy_times(i->raw_job->disk_fd, dfd, COPY_CRTIME); + (void) copy_xattr(i->raw_job->disk_fd, dfd, 0); + + dfd = safe_close(dfd); + + r = install_file(AT_FDCWD, tp, + AT_FDCWD, p, + (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) + return log_error_errno(errno, "Failed to move local image into place '%s': %m", p); + + tp = mfree(tp); + + log_info("Created new local image '%s'.", i->local); + + if (FLAGS_SET(i->flags, PULL_SETTINGS)) { + r = raw_pull_copy_auxiliary_file(i, ".nspawn", &i->settings_path); + if (r < 0) + return r; + } + + if (FLAGS_SET(i->flags, PULL_ROOTHASH)) { + r = raw_pull_copy_auxiliary_file(i, ".roothash", &i->roothash_path); + if (r < 0) + return r; + } + + if (FLAGS_SET(i->flags, PULL_ROOTHASH_SIGNATURE)) { + r = raw_pull_copy_auxiliary_file(i, ".roothash.p7s", &i->roothash_signature_path); + if (r < 0) + return r; + } + + if (FLAGS_SET(i->flags, PULL_VERITY)) { + r = raw_pull_copy_auxiliary_file(i, ".verity", &i->verity_path); + if (r < 0) + return r; + } + + return 0; +} + +static bool raw_pull_is_done(RawPull *i) { + assert(i); + assert(i->raw_job); + + if (!PULL_JOB_IS_COMPLETE(i->raw_job)) + return false; + if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job)) + return false; + if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job)) + return false; + if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job)) + return false; + if (i->roothash_job && !PULL_JOB_IS_COMPLETE(i->roothash_job)) + return false; + if (i->roothash_signature_job && !PULL_JOB_IS_COMPLETE(i->roothash_signature_job)) + return false; + if (i->verity_job && !PULL_JOB_IS_COMPLETE(i->verity_job)) + return false; + + return true; +} + +static int raw_pull_rename_auxiliary_file( + RawPull *i, + const char *suffix, + char **temp_path, + char **path) { + + int r; + + assert(i); + assert(path); + assert(temp_path); + assert(*temp_path); + assert(suffix); + + /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should + * incorporate it in the file name if we can */ + *path = mfree(*path); + r = raw_pull_determine_path(i, suffix, path); + if (r < 0) + return r; + + r = install_file( + AT_FDCWD, *temp_path, + AT_FDCWD, *path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) + return log_error_errno(r, "Failed to move '%s' into place: %m", *path); + + *temp_path = mfree(*temp_path); + return 1; +} + +static void raw_pull_job_on_finished(PullJob *j) { + RawPull *i; + PullJob *jj; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + if (j->error != 0) { + /* Only the main job and the checksum job are fatal if they fail. The other fails are just + * "decoration", that we'll download if we can. The signature job isn't fatal here because we + * might not actually need it in case Suse style signatures are used, that are inline in the + * checksum file. */ + + if (j == i->raw_job) { + if (j->error == ENOMEDIUM) /* HTTP 404 */ + r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); + else + r = log_error_errno(j->error, "Failed to retrieve image file."); + goto finish; + } else if (j == i->checksum_job) { + r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); + goto finish; + } else if (j == i->signature_job) + log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url); + else if (j == i->settings_job) + log_info_errno(j->error, "Settings file could not be retrieved, proceeding without."); + else if (j == i->roothash_job) + log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without."); + else if (j == i->roothash_signature_job) + log_info_errno(j->error, "Root hash signature file could not be retrieved, proceeding without."); + else if (j == i->verity_job) + log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without."); + else + assert_not_reached(); + } + + /* This is invoked if either the download completed successfully, or the download was skipped because + * we already have the etag. In this case ->etag_exists is true. + * + * We only do something when we got all files */ + + if (!raw_pull_is_done(i)) + return; + + if (i->signature_job && i->signature_job->error != 0) { + VerificationStyle style; + PullJob *verify_job; + + /* The signature job failed. Let's see if we actually need it */ + + verify_job = i->checksum_job ?: i->raw_job; /* if the checksum job doesn't exist this must be + * because the main job is the checksum file + * itself */ + + assert(verify_job); + + r = verification_style_from_url(verify_job->url, &style); + if (r < 0) { + log_error_errno(r, "Failed to determine verification style from checksum URL: %m"); + goto finish; + } + + if (style == VERIFICATION_PER_DIRECTORY) { /* A failed signature file download only matters + * in per-directory verification mode, since only + * then the signature is detached, and thus a file + * of its own. */ + r = log_error_errno(i->signature_job->error, + "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); + goto finish; + } + } + + /* Let's close these auxiliary files now, we don't need access to them anymore. */ + FOREACH_POINTER(jj, i->settings_job, i->roothash_job, i->roothash_signature_job, i->verity_job) + pull_job_close_disk_fd(jj); + + if (!i->raw_job->etag_exists) { + raw_pull_report_progress(i, RAW_VERIFYING); + + r = pull_verify(i->verify, + i->checksum, + i->raw_job, + i->checksum_job, + i->signature_job, + i->settings_job, + i->roothash_job, + i->roothash_signature_job, + i->verity_job); + if (r < 0) + goto finish; + } + + if (i->flags & PULL_DIRECT) { + assert(!i->settings_job); + assert(!i->roothash_job); + assert(!i->roothash_signature_job); + assert(!i->verity_job); + + raw_pull_report_progress(i, RAW_FINALIZING); + + if (i->local) { + r = install_file(AT_FDCWD, i->local, + AT_FDCWD, NULL, + ((i->flags & PULL_READ_ONLY) && i->offset == UINT64_MAX ? INSTALL_READ_ONLY : 0) | + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) { + log_error_errno(r, "Failed to finalize raw file to '%s': %m", i->local); + goto finish; + } + } + } else { + r = raw_pull_determine_path(i, ".raw", &i->final_path); + if (r < 0) + goto finish; + + if (!i->raw_job->etag_exists) { + /* This is a new download, verify it, and move it into place */ + + assert(i->temp_path); + assert(i->final_path); + + raw_pull_report_progress(i, RAW_UNPACKING); + + r = raw_pull_maybe_convert_qcow2(i); + if (r < 0) + goto finish; + + raw_pull_report_progress(i, RAW_FINALIZING); + + r = install_file(AT_FDCWD, i->temp_path, + AT_FDCWD, i->final_path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) { + log_error_errno(r, "Failed to move raw file to '%s': %m", i->final_path); + goto finish; + } + + i->temp_path = mfree(i->temp_path); + + if (i->settings_job && + i->settings_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path); + if (r < 0) + goto finish; + } + + if (i->roothash_job && + i->roothash_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path); + if (r < 0) + goto finish; + } + + if (i->roothash_signature_job && + i->roothash_signature_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".roothash.p7s", &i->roothash_signature_temp_path, &i->roothash_signature_path); + if (r < 0) + goto finish; + } + + if (i->verity_job && + i->verity_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".verity", &i->verity_temp_path, &i->verity_path); + if (r < 0) + goto finish; + } + } + + raw_pull_report_progress(i, RAW_COPYING); + + r = raw_pull_make_local_copy(i); + if (r < 0) + goto finish; + } + + r = 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); +} + +static int raw_pull_job_on_open_disk_generic( + RawPull *i, + PullJob *j, + const char *extra, + char **temp_path /* input + output */) { + + int r; + + assert(i); + assert(j); + assert(extra); + assert(temp_path); + + assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + + if (!*temp_path) { + r = tempfn_random_child(i->image_root, extra, temp_path); + if (r < 0) + return log_oom(); + } + + (void) mkdir_parents_label(*temp_path, 0700); + + j->disk_fd = open(*temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (j->disk_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", *temp_path); + + return 0; +} + +static int raw_pull_job_on_open_disk_raw(PullJob *j) { + RawPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->raw_job == j); + assert(j->disk_fd < 0); + + if (i->flags & PULL_DIRECT) { + + if (!i->local) { /* If no local name specified, the pull job will write its data to stdout */ + j->disk_fd = STDOUT_FILENO; + j->close_disk_fd = false; + return 0; + } + + (void) mkdir_parents_label(i->local, 0700); + + j->disk_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664); + if (j->disk_fd < 0) + return log_error_errno(errno, "Failed to open destination '%s': %m", i->local); + + if (i->offset == UINT64_MAX) + (void) import_set_nocow_and_log(j->disk_fd, i->local); + + } else { + r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path); + if (r < 0) + return r; + + assert(i->offset == UINT64_MAX); + (void) import_set_nocow_and_log(j->disk_fd, i->temp_path); + } + + return 0; +} + +static int raw_pull_job_on_open_disk_settings(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->settings_job == j); + + return raw_pull_job_on_open_disk_generic(i, j, "settings", &i->settings_temp_path); +} + +static int raw_pull_job_on_open_disk_roothash(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->roothash_job == j); + + return raw_pull_job_on_open_disk_generic(i, j, "roothash", &i->roothash_temp_path); +} + +static int raw_pull_job_on_open_disk_roothash_signature(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->roothash_signature_job == j); + + return raw_pull_job_on_open_disk_generic(i, j, "roothash.p7s", &i->roothash_signature_temp_path); +} + +static int raw_pull_job_on_open_disk_verity(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->verity_job == j); + + return raw_pull_job_on_open_disk_generic(i, j, "verity", &i->verity_temp_path); +} + +static void raw_pull_job_on_progress(PullJob *j) { + RawPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + raw_pull_report_progress(i, RAW_DOWNLOADING); +} + +int raw_pull_start( + RawPull *i, + const char *url, + const char *local, + uint64_t offset, + uint64_t size_max, + PullFlags flags, + ImportVerify verify, + const char *checksum) { + + PullJob *j; + int r; + + assert(i); + assert(url); + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); + assert(!(flags & ~PULL_FLAGS_MASK_RAW)); + assert(offset == UINT64_MAX || FLAGS_SET(flags, PULL_DIRECT)); + assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !(flags & PULL_DIRECT)); + assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !checksum); + + if (!http_url_is_valid(url) && !file_url_is_valid(url)) + return -EINVAL; + + if (local && !pull_validate_local(local, flags)) + return -EINVAL; + + if (i->raw_job) + return -EBUSY; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + + r = free_and_strdup(&i->checksum, checksum); + if (r < 0) + return r; + + i->flags = flags; + i->verify = verify; + + /* Queue job for the image itself */ + r = pull_job_new(&i->raw_job, url, i->glue, i); + if (r < 0) + return r; + + i->raw_job->on_finished = raw_pull_job_on_finished; + i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw; + + if (checksum) + i->raw_job->calc_checksum = true; + else if (verify != IMPORT_VERIFY_NO) { + /* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its + * signature, which we let gpg verify instead. */ + + r = pull_url_needs_checksum(url); + if (r < 0) + return r; + + i->raw_job->calc_checksum = r; + i->raw_job->force_memory = true; /* make sure this is both written to disk if that's + * requested and into memory, since we need to verify it */ + } + + if (size_max != UINT64_MAX) + i->raw_job->uncompressed_max = size_max; + if (offset != UINT64_MAX) + i->raw_job->offset = i->offset = offset; + + if (!FLAGS_SET(flags, PULL_DIRECT)) { + r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags); + if (r < 0) + return r; + } + + r = pull_make_verification_jobs( + &i->checksum_job, + &i->signature_job, + verify, + i->checksum, + url, + i->glue, + raw_pull_job_on_finished, + i); + if (r < 0) + return r; + + if (FLAGS_SET(flags, PULL_SETTINGS)) { + r = pull_make_auxiliary_job( + &i->settings_job, + url, + raw_strip_suffixes, + ".nspawn", + verify, + i->glue, + raw_pull_job_on_open_disk_settings, + raw_pull_job_on_finished, + i); + if (r < 0) + return r; + } + + if (FLAGS_SET(flags, PULL_ROOTHASH)) { + r = pull_make_auxiliary_job( + &i->roothash_job, + url, + raw_strip_suffixes, + ".roothash", + verify, + i->glue, + raw_pull_job_on_open_disk_roothash, + raw_pull_job_on_finished, + i); + if (r < 0) + return r; + } + + if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) { + r = pull_make_auxiliary_job( + &i->roothash_signature_job, + url, + raw_strip_suffixes, + ".roothash.p7s", + verify, + i->glue, + raw_pull_job_on_open_disk_roothash_signature, + raw_pull_job_on_finished, + i); + if (r < 0) + return r; + } + + if (FLAGS_SET(flags, PULL_VERITY)) { + r = pull_make_auxiliary_job( + &i->verity_job, + url, + raw_strip_suffixes, + ".verity", + verify, + i->glue, + raw_pull_job_on_open_disk_verity, + raw_pull_job_on_finished, + i); + if (r < 0) + return r; + } + + FOREACH_POINTER(j, + i->raw_job, + i->checksum_job, + i->signature_job, + i->settings_job, + i->roothash_job, + i->roothash_signature_job, + i->verity_job) { + + if (!j) + continue; + + j->on_progress = raw_pull_job_on_progress; + j->sync = FLAGS_SET(flags, PULL_SYNC); + + r = pull_job_begin(j); + if (r < 0) + return r; + } + + return 0; +} diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h new file mode 100644 index 0000000..b39e4e2 --- /dev/null +++ b/src/import/pull-raw.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "sd-event.h" + +#include "import-util.h" +#include "macro.h" +#include "pull-common.h" + +typedef struct RawPull RawPull; + +typedef void (*RawPullFinished)(RawPull *pull, int error, void *userdata); + +int raw_pull_new(RawPull **pull, sd_event *event, const char *image_root, RawPullFinished on_finished, void *userdata); +RawPull* raw_pull_unref(RawPull *pull); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref); + +int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, PullFlags flags, ImportVerify verify, const char *checksum); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c new file mode 100644 index 0000000..fd866fc --- /dev/null +++ b/src/import/pull-tar.c @@ -0,0 +1,679 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <curl/curl.h> +#include <sys/prctl.h> + +#include "sd-daemon.h" + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "curl-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-util.h" +#include "install-file.h" +#include "macro.h" +#include "mkdir-label.h" +#include "path-util.h" +#include "process-util.h" +#include "pull-common.h" +#include "pull-job.h" +#include "pull-tar.h" +#include "rm-rf.h" +#include "string-util.h" +#include "strv.h" +#include "tmpfile-util.h" +#include "user-util.h" +#include "utf8.h" +#include "util.h" +#include "web-util.h" + +typedef enum TarProgress { + TAR_DOWNLOADING, + TAR_VERIFYING, + TAR_FINALIZING, + TAR_COPYING, +} TarProgress; + +struct TarPull { + sd_event *event; + CurlGlue *glue; + + PullFlags flags; + ImportVerify verify; + char *image_root; + + PullJob *tar_job; + PullJob *checksum_job; + PullJob *signature_job; + PullJob *settings_job; + + TarPullFinished on_finished; + void *userdata; + + char *local; + + pid_t tar_pid; + + char *final_path; + char *temp_path; + + char *settings_path; + char *settings_temp_path; + + char *checksum; +}; + +TarPull* tar_pull_unref(TarPull *i) { + if (!i) + return NULL; + + if (i->tar_pid > 1) + sigkill_wait(i->tar_pid); + + pull_job_unref(i->tar_job); + pull_job_unref(i->checksum_job); + pull_job_unref(i->signature_job); + pull_job_unref(i->settings_job); + + curl_glue_unref(i->glue); + sd_event_unref(i->event); + + rm_rf_subvolume_and_free(i->temp_path); + unlink_and_free(i->settings_temp_path); + + free(i->final_path); + free(i->settings_path); + free(i->image_root); + free(i->local); + free(i->checksum); + + return mfree(i); +} + +int tar_pull_new( + TarPull **ret, + sd_event *event, + const char *image_root, + TarPullFinished on_finished, + void *userdata) { + + _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL; + _cleanup_(sd_event_unrefp) sd_event *e = NULL; + _cleanup_(tar_pull_unrefp) TarPull *i = NULL; + _cleanup_free_ char *root = NULL; + int r; + + assert(ret); + + root = strdup(image_root ?: "/var/lib/machines"); + if (!root) + return -ENOMEM; + + if (event) + e = sd_event_ref(event); + else { + r = sd_event_default(&e); + if (r < 0) + return r; + } + + r = curl_glue_new(&g, e); + if (r < 0) + return r; + + i = new(TarPull, 1); + if (!i) + return -ENOMEM; + + *i = (TarPull) { + .on_finished = on_finished, + .userdata = userdata, + .image_root = TAKE_PTR(root), + .event = TAKE_PTR(e), + .glue = TAKE_PTR(g), + }; + + i->glue->on_finished = pull_job_curl_on_finished; + i->glue->userdata = i; + + *ret = TAKE_PTR(i); + + return 0; +} + +static void tar_pull_report_progress(TarPull *i, TarProgress p) { + unsigned percent; + + assert(i); + + switch (p) { + + case TAR_DOWNLOADING: { + unsigned remain = 85; + + percent = 0; + + if (i->checksum_job) { + percent += i->checksum_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->signature_job) { + percent += i->signature_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->settings_job) { + percent += i->settings_job->progress_percent * 5 / 100; + remain -= 5; + } + + if (i->tar_job) + percent += i->tar_job->progress_percent * remain / 100; + break; + } + + case TAR_VERIFYING: + percent = 85; + break; + + case TAR_FINALIZING: + percent = 90; + break; + + case TAR_COPYING: + percent = 95; + break; + + default: + assert_not_reached(); + } + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_debug("Combined progress %u%%", percent); +} + +static int tar_pull_determine_path( + TarPull *i, + const char *suffix, + char **field /* input + output (!) */) { + int r; + + assert(i); + assert(field); + + if (*field) + return 0; + + assert(i->tar_job); + + r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field); + if (r < 0) + return log_oom(); + + return 1; +} + +static int tar_pull_make_local_copy(TarPull *i) { + _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL; + const char *p; + int r; + + assert(i); + assert(i->tar_job); + + if (!i->local) + return 0; + + assert(i->final_path); + + p = prefix_roota(i->image_root, i->local); + + r = tempfn_random(p, NULL, &t); + if (r < 0) + return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p); + + if (i->flags & PULL_BTRFS_SUBVOL) + r = btrfs_subvol_snapshot( + i->final_path, + t, + (i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)| + BTRFS_SNAPSHOT_FALLBACK_COPY| + BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| + BTRFS_SNAPSHOT_RECURSIVE); + else + r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS); + if (r < 0) + return log_error_errno(r, "Failed to create local image: %m"); + + r = install_file(AT_FDCWD, t, + AT_FDCWD, p, + (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); + if (r < 0) + return log_error_errno(r, "Failed to install local image '%s': %m", p); + + t = mfree(t); + + log_info("Created new local image '%s'.", i->local); + + if (FLAGS_SET(i->flags, PULL_SETTINGS)) { + const char *local_settings; + assert(i->settings_job); + + r = tar_pull_determine_path(i, ".nspawn", &i->settings_path); + if (r < 0) + return r; + + local_settings = strjoina(i->image_root, "/", i->local, ".nspawn"); + + r = copy_file_atomic( + i->settings_path, + local_settings, + 0664, + 0, 0, + COPY_REFLINK | + (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0)); + if (r == -EEXIST) + log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings); + else if (r == -ENOENT) + log_debug_errno(r, "Skipping creation of settings file, since none was found."); + else if (r < 0) + log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings); + else + log_info("Created new settings file %s.", local_settings); + } + + return 0; +} + +static bool tar_pull_is_done(TarPull *i) { + assert(i); + assert(i->tar_job); + + if (!PULL_JOB_IS_COMPLETE(i->tar_job)) + return false; + if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job)) + return false; + if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job)) + return false; + if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job)) + return false; + + return true; +} + +static void tar_pull_job_on_finished(PullJob *j) { + TarPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + if (j->error != 0) { + if (j == i->tar_job) { + if (j->error == ENOMEDIUM) /* HTTP 404 */ + r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); + else + r = log_error_errno(j->error, "Failed to retrieve image file."); + goto finish; + } else if (j == i->checksum_job) { + r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); + goto finish; + } else if (j == i->signature_job) + log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url); + else if (j == i->settings_job) + log_info_errno(j->error, "Settings file could not be retrieved, proceeding without."); + else + assert("unexpected job"); + } + + /* This is invoked if either the download completed successfully, or the download was skipped because + * we already have the etag. */ + + if (!tar_pull_is_done(i)) + return; + + if (i->signature_job && i->signature_job->error != 0) { + VerificationStyle style; + + assert(i->checksum_job); + + r = verification_style_from_url(i->checksum_job->url, &style); + if (r < 0) { + log_error_errno(r, "Failed to determine verification style from checksum URL: %m"); + goto finish; + } + + if (style == VERIFICATION_PER_DIRECTORY) { /* A failed signature file download only matters + * in per-directory verification mode, since only + * then the signature is detached, and thus a file + * of its own. */ + r = log_error_errno(i->signature_job->error, + "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); + goto finish; + } + } + + pull_job_close_disk_fd(i->tar_job); + pull_job_close_disk_fd(i->settings_job); + + if (i->tar_pid > 0) { + r = wait_for_terminate_and_check("tar", TAKE_PID(i->tar_pid), WAIT_LOG); + if (r < 0) + goto finish; + if (r != EXIT_SUCCESS) { + r = -EIO; + goto finish; + } + } + + if (!i->tar_job->etag_exists) { + /* This is a new download, verify it, and move it into place */ + + tar_pull_report_progress(i, TAR_VERIFYING); + + r = pull_verify(i->verify, + i->checksum, + i->tar_job, + i->checksum_job, + i->signature_job, + i->settings_job, + /* roothash_job = */ NULL, + /* roothash_signature_job = */ NULL, + /* verity_job = */ NULL); + if (r < 0) + goto finish; + } + + if (i->flags & PULL_DIRECT) { + assert(!i->settings_job); + assert(i->local); + assert(!i->temp_path); + + tar_pull_report_progress(i, TAR_FINALIZING); + + r = import_mangle_os_tree(i->local); + if (r < 0) + goto finish; + + r = install_file( + AT_FDCWD, i->local, + AT_FDCWD, NULL, + (i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 | + (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); + if (r < 0) { + log_error_errno(r, "Failed to finalize '%s': %m", i->local); + goto finish; + } + } else { + r = tar_pull_determine_path(i, NULL, &i->final_path); + if (r < 0) + goto finish; + + if (!i->tar_job->etag_exists) { + /* This is a new download, verify it, and move it into place */ + + assert(i->temp_path); + assert(i->final_path); + + tar_pull_report_progress(i, TAR_FINALIZING); + + r = import_mangle_os_tree(i->temp_path); + if (r < 0) + goto finish; + + r = install_file( + AT_FDCWD, i->temp_path, + AT_FDCWD, i->final_path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); + if (r < 0) { + log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path); + goto finish; + } + + i->temp_path = mfree(i->temp_path); + + if (i->settings_job && + i->settings_job->error == 0) { + + /* Also move the settings file into place, if it exists. Note that we do so only if we also + * moved the tar file in place, to keep things strictly in sync. */ + assert(i->settings_temp_path); + + /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and + * we should incorporate it in the file name if we can */ + i->settings_path = mfree(i->settings_path); + + r = tar_pull_determine_path(i, ".nspawn", &i->settings_path); + if (r < 0) + goto finish; + + r = install_file( + AT_FDCWD, i->settings_temp_path, + AT_FDCWD, i->settings_path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) { + log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path); + goto finish; + } + + i->settings_temp_path = mfree(i->settings_temp_path); + } + } + + tar_pull_report_progress(i, TAR_COPYING); + + r = tar_pull_make_local_copy(i); + if (r < 0) + goto finish; + } + + r = 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); +} + +static int tar_pull_job_on_open_disk_tar(PullJob *j) { + const char *where; + TarPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->tar_job == j); + assert(i->tar_pid <= 0); + + if (i->flags & PULL_DIRECT) + where = i->local; + else { + if (!i->temp_path) { + r = tempfn_random_child(i->image_root, "tar", &i->temp_path); + if (r < 0) + return log_oom(); + } + + where = i->temp_path; + } + + (void) mkdir_parents_label(where, 0700); + + if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE)) + (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + if (i->flags & PULL_BTRFS_SUBVOL) + r = btrfs_subvol_make_fallback(where, 0755); + else + r = RET_NERRNO(mkdir(where, 0755)); + if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise, + * because in that case our temporary path collided */ + r = 0; + if (r < 0) + return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where); + if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */ + if (!(i->flags & PULL_DIRECT)) + (void) import_assign_pool_quota_and_warn(i->image_root); + (void) import_assign_pool_quota_and_warn(where); + } + + j->disk_fd = import_fork_tar_x(where, &i->tar_pid); + if (j->disk_fd < 0) + return j->disk_fd; + + return 0; +} + +static int tar_pull_job_on_open_disk_settings(PullJob *j) { + TarPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->settings_job == j); + + if (!i->settings_temp_path) { + r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path); + if (r < 0) + return log_oom(); + } + + (void) mkdir_parents_label(i->settings_temp_path, 0700); + + j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (j->disk_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path); + + return 0; +} + +static void tar_pull_job_on_progress(PullJob *j) { + TarPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + tar_pull_report_progress(i, TAR_DOWNLOADING); +} + +int tar_pull_start( + TarPull *i, + const char *url, + const char *local, + PullFlags flags, + ImportVerify verify, + const char *checksum) { + + PullJob *j; + int r; + + assert(i); + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); + assert(!(flags & ~PULL_FLAGS_MASK_TAR)); + assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT)); + assert(!(flags & PULL_SETTINGS) || !checksum); + + if (!http_url_is_valid(url) && !file_url_is_valid(url)) + return -EINVAL; + + if (local && !pull_validate_local(local, flags)) + return -EINVAL; + + if (i->tar_job) + return -EBUSY; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + + r = free_and_strdup(&i->checksum, checksum); + if (r < 0) + return r; + + i->flags = flags; + i->verify = verify; + + /* Set up download job for TAR file */ + r = pull_job_new(&i->tar_job, url, i->glue, i); + if (r < 0) + return r; + + i->tar_job->on_finished = tar_pull_job_on_finished; + i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar; + i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE); + + if (!FLAGS_SET(flags, PULL_DIRECT)) { + r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags); + if (r < 0) + return r; + } + + /* Set up download of checksum/signature files */ + r = pull_make_verification_jobs( + &i->checksum_job, + &i->signature_job, + verify, + checksum, + url, + i->glue, + tar_pull_job_on_finished, + i); + if (r < 0) + return r; + + /* Set up download job for the settings file (.nspawn) */ + if (FLAGS_SET(flags, PULL_SETTINGS)) { + r = pull_make_auxiliary_job( + &i->settings_job, + url, + tar_strip_suffixes, + ".nspawn", + verify, + i->glue, + tar_pull_job_on_open_disk_settings, + tar_pull_job_on_finished, + i); + if (r < 0) + return r; + } + + FOREACH_POINTER(j, + i->tar_job, + i->checksum_job, + i->signature_job, + i->settings_job) { + + if (!j) + continue; + + j->on_progress = tar_pull_job_on_progress; + j->sync = FLAGS_SET(flags, PULL_SYNC); + + r = pull_job_begin(j); + if (r < 0) + return r; + } + + return 0; +} diff --git a/src/import/pull-tar.h b/src/import/pull-tar.h new file mode 100644 index 0000000..e54c01c --- /dev/null +++ b/src/import/pull-tar.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "sd-event.h" + +#include "import-util.h" +#include "macro.h" +#include "pull-common.h" + +typedef struct TarPull TarPull; + +typedef void (*TarPullFinished)(TarPull *pull, int error, void *userdata); + +int tar_pull_new(TarPull **pull, sd_event *event, const char *image_root, TarPullFinished on_finished, void *userdata); +TarPull* tar_pull_unref(TarPull *pull); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref); + +int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify, const char *checksum); diff --git a/src/import/pull.c b/src/import/pull.c new file mode 100644 index 0000000..2cf0cca --- /dev/null +++ b/src/import/pull.c @@ -0,0 +1,555 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <getopt.h> +#include <locale.h> + +#include "sd-event.h" +#include "sd-id128.h" + +#include "alloc-util.h" +#include "discover-image.h" +#include "env-util.h" +#include "hexdecoct.h" +#include "hostname-util.h" +#include "import-common.h" +#include "import-util.h" +#include "io-util.h" +#include "main-func.h" +#include "parse-argument.h" +#include "parse-util.h" +#include "pull-raw.h" +#include "pull-tar.h" +#include "signal-util.h" +#include "string-util.h" +#include "terminal-util.h" +#include "verbs.h" +#include "web-util.h" + +static const char *arg_image_root = "/var/lib/machines"; +static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; +static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY | PULL_BTRFS_SUBVOL | PULL_BTRFS_QUOTA | PULL_CONVERT_QCOW2 | PULL_SYNC; +static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; +static char *arg_checksum = NULL; + +STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep); + +static int normalize_local(const char *local, const char *url, char **ret) { + _cleanup_free_ char *ll = NULL; + int r; + + if (arg_pull_flags & PULL_DIRECT) { + + if (!local) + log_debug("Writing downloaded data to STDOUT."); + else { + if (!path_is_absolute(local)) { + ll = path_join(arg_image_root, local); + if (!ll) + return log_oom(); + + local = ll; + } + + if (!path_is_valid(local)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local path name '%s' is not valid.", local); + } + + } else if (local) { + + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local image name '%s' is not valid.", + local); + + if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) { + r = image_find(IMAGE_MACHINE, local, NULL, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else + return log_error_errno(SYNTHETIC_ERRNO(EEXIST), + "Image '%s' already exists.", + local); + } + } + + if (!ll && local) { + ll = strdup(local); + if (!ll) + return log_oom(); + } + + if (ll) { + if (arg_offset != UINT64_MAX) + log_info("Pulling '%s', saving at offset %" PRIu64 " in '%s'.", url, arg_offset, ll); + else + log_info("Pulling '%s', saving as '%s'.", url, ll); + } else + log_info("Pulling '%s'.", url); + + *ret = TAKE_PTR(ll); + return 0; +} + +static void on_tar_finished(TarPull *pull, int error, void *userdata) { + sd_event *event = userdata; + assert(pull); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int pull_tar(int argc, char *argv[], void *userdata) { + _cleanup_free_ char *ll = NULL, *normalized = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(tar_pull_unrefp) TarPull *pull = NULL; + const char *url, *local; + int r; + + url = argv[1]; + if (!http_url_is_valid(url) && !file_url_is_valid(url)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else { + _cleanup_free_ char *l = NULL; + + r = import_url_last_component(url, &l); + if (r < 0) + return log_error_errno(r, "Failed to get final component of URL: %m"); + + r = tar_strip_suffixes(l, &ll); + if (r < 0) + return log_oom(); + + local = ll; + } + + if (!local && FLAGS_SET(arg_pull_flags, PULL_DIRECT)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Pulling tar images to STDOUT is not supported."); + + r = normalize_local(local, url, &normalized); + if (r < 0) + return r; + + r = import_allocate_event_with_signals(&event); + if (r < 0) + return r; + + if (!FLAGS_SET(arg_pull_flags, PULL_SYNC)) + log_info("File system synchronization on completion is off."); + + r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate puller: %m"); + + r = tar_pull_start( + pull, + url, + normalized, + arg_pull_flags & PULL_FLAGS_MASK_TAR, + arg_verify, + arg_checksum); + if (r < 0) + return log_error_errno(r, "Failed to pull image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawPull *pull, int error, void *userdata) { + sd_event *event = userdata; + assert(pull); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int pull_raw(int argc, char *argv[], void *userdata) { + _cleanup_free_ char *ll = NULL, *normalized = NULL; + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(raw_pull_unrefp) RawPull *pull = NULL; + const char *url, *local; + int r; + + url = argv[1]; + if (!http_url_is_valid(url) && !file_url_is_valid(url)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else { + _cleanup_free_ char *l = NULL; + + r = import_url_last_component(url, &l); + if (r < 0) + return log_error_errno(r, "Failed to get final component of URL: %m"); + + r = raw_strip_suffixes(l, &ll); + if (r < 0) + return log_oom(); + + local = ll; + } + + r = normalize_local(local, url, &normalized); + if (r < 0) + return r; + + r = import_allocate_event_with_signals(&event); + if (r < 0) + return r; + + if (!FLAGS_SET(arg_pull_flags, PULL_SYNC)) + log_info("File system synchronization on completion is off."); + r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate puller: %m"); + + r = raw_pull_start( + pull, + url, + normalized, + arg_offset, + arg_size_max, + arg_pull_flags & PULL_FLAGS_MASK_RAW, + arg_verify, + arg_checksum); + if (r < 0) + return log_error_errno(r, "Failed to pull image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%1$s [OPTIONS...] {COMMAND} ...\n" + "\n%4$sDownload container or virtual machine images.%5$s\n" + "\n%2$sCommands:%3$s\n" + " tar URL [NAME] Download a TAR image\n" + " raw URL [NAME] Download a RAW image\n" + "\n%2$sOptions:%3$s\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --force Force creation of image\n" + " --verify=MODE Verify downloaded image, one of: 'no',\n" + " 'checksum', 'signature' or literal SHA256 hash\n" + " --settings=BOOL Download settings file with image\n" + " --roothash=BOOL Download root hash file with image\n" + " --roothash-signature=BOOL\n" + " Download root hash signature file with image\n" + " --verity=BOOL Download verity file with image\n" + " --image-root=PATH Image root directory\n\n" + " --read-only Create a read-only image\n" + " --direct Download directly to specified file\n" + " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n" + " instead of a directory\n" + " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n" + " subvolume\n" + " --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n" + " regular disk images\n" + " --sync=BOOL Controls whether to sync() before completing\n" + " --offset=BYTES Offset to seek to in destination\n" + " --size-max=BYTES Maximum number of bytes to write to destination\n", + program_invocation_short_name, + ansi_underline(), + ansi_normal(), + ansi_highlight(), + ansi_normal()); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORCE, + ARG_IMAGE_ROOT, + ARG_VERIFY, + ARG_SETTINGS, + ARG_ROOTHASH, + ARG_ROOTHASH_SIGNATURE, + ARG_VERITY, + ARG_READ_ONLY, + ARG_DIRECT, + ARG_BTRFS_SUBVOL, + ARG_BTRFS_QUOTA, + ARG_CONVERT_QCOW2, + ARG_SYNC, + ARG_OFFSET, + ARG_SIZE_MAX, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "force", no_argument, NULL, ARG_FORCE }, + { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, + { "verify", required_argument, NULL, ARG_VERIFY }, + { "settings", required_argument, NULL, ARG_SETTINGS }, + { "roothash", required_argument, NULL, ARG_ROOTHASH }, + { "roothash-signature", required_argument, NULL, ARG_ROOTHASH_SIGNATURE }, + { "verity", required_argument, NULL, ARG_VERITY }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "direct", no_argument, NULL, ARG_DIRECT }, + { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL }, + { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA }, + { "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 }, + { "sync", required_argument, NULL, ARG_SYNC }, + { "offset", required_argument, NULL, ARG_OFFSET }, + { "size-max", required_argument, NULL, ARG_SIZE_MAX }, + {} + }; + + int c, r; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_FORCE: + arg_pull_flags |= PULL_FORCE; + break; + + case ARG_IMAGE_ROOT: + arg_image_root = optarg; + break; + + case ARG_VERIFY: { + ImportVerify v; + + v = import_verify_from_string(optarg); + if (v < 0) { + _cleanup_free_ void *h = NULL; + char *hh; + size_t n; + + /* If this is not a valid verification mode, maybe it's a literally specified + * SHA256 hash? We can handle that too... */ + + r = unhexmem(optarg, (size_t) -1, &h, &n); + if (r < 0 || n == 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Invalid verification setting: %s", optarg); + if (n != 32) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2); + + hh = hexmem(h, n); /* bring into canonical (lowercase) form */ + if (!hh) + return log_oom(); + + free_and_replace(arg_checksum, hh); + arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY); + arg_verify = _IMPORT_VERIFY_INVALID; + } else + arg_verify = v; + + break; + } + + case ARG_SETTINGS: + r = parse_boolean_argument("--settings=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_SETTINGS, r); + break; + + case ARG_ROOTHASH: + r = parse_boolean_argument("--roothash=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_ROOTHASH, r); + + /* If we were asked to turn off the root hash, implicitly also turn off the root hash signature */ + if (!r) + SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, false); + break; + + case ARG_ROOTHASH_SIGNATURE: + r = parse_boolean_argument("--roothash-signature=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, r); + break; + + case ARG_VERITY: + r = parse_boolean_argument("--verity=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_VERITY, r); + break; + + case ARG_READ_ONLY: + arg_pull_flags |= PULL_READ_ONLY; + break; + + case ARG_DIRECT: + arg_pull_flags |= PULL_DIRECT; + arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY); + break; + + case ARG_BTRFS_SUBVOL: + r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r); + break; + + case ARG_BTRFS_QUOTA: + r = parse_boolean_argument("--btrfs-quota=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r); + break; + + case ARG_CONVERT_QCOW2: + r = parse_boolean_argument("--convert-qcow2=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_CONVERT_QCOW2, r); + break; + + case ARG_SYNC: + r = parse_boolean_argument("--sync=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_SYNC, r); + break; + + case ARG_OFFSET: { + uint64_t u; + + r = safe_atou64(optarg, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg); + + arg_offset = u; + break; + } + + case ARG_SIZE_MAX: { + uint64_t u; + + r = parse_size(optarg, 1024, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg); + + arg_size_max = u; + break; + } + + case '?': + return -EINVAL; + + default: + assert_not_reached(); + } + + /* Make sure offset+size is still in the valid range if both set */ + if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX && + ((arg_size_max > (UINT64_MAX - arg_offset)) || + !FILE_SIZE_VALID(arg_offset + arg_size_max))) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range."); + + if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_pull_flags, PULL_DIRECT)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); + + if (arg_checksum && (arg_pull_flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) != 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded."); + + return 1; +} + +static void parse_env(void) { + int r; + + /* Let's make these relatively low-level settings also controllable via env vars. User can then set + * them for systemd-importd.service if they like to tweak behaviour */ + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL"); + if (r >= 0) + SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA"); + if (r >= 0) + SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_SYNC"); + if (r >= 0) + SET_FLAG(arg_pull_flags, PULL_SYNC, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m"); +} + +static int pull_main(int argc, char *argv[]) { + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, pull_tar }, + { "raw", 2, 3, 0, pull_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +static int run(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + parse_env(); + + r = parse_argv(argc, argv); + if (r <= 0) + return r; + + (void) ignore_signals(SIGPIPE); + + return pull_main(argc, argv); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/qcow2-util.c b/src/import/qcow2-util.c new file mode 100644 index 0000000..5a7232d --- /dev/null +++ b/src/import/qcow2-util.c @@ -0,0 +1,334 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <zlib.h> + +#include "alloc-util.h" +#include "btrfs-util.h" +#include "qcow2-util.h" +#include "sparse-endian.h" +#include "util.h" + +#define QCOW2_MAGIC 0x514649fb + +#define QCOW2_COPIED (1ULL << 63) +#define QCOW2_COMPRESSED (1ULL << 62) +#define QCOW2_ZERO (1ULL << 0) + +typedef struct _packed_ Header { + be32_t magic; + be32_t version; + + be64_t backing_file_offset; + be32_t backing_file_size; + + be32_t cluster_bits; + be64_t size; + be32_t crypt_method; + + be32_t l1_size; + be64_t l1_table_offset; + + be64_t refcount_table_offset; + be32_t refcount_table_clusters; + + be32_t nb_snapshots; + be64_t snapshots_offset; + + /* The remainder is only present on QCOW3 */ + be64_t incompatible_features; + be64_t compatible_features; + be64_t autoclear_features; + + be32_t refcount_order; + be32_t header_length; +} Header; + +#define HEADER_MAGIC(header) be32toh((header)->magic) +#define HEADER_VERSION(header) be32toh((header)->version) +#define HEADER_CLUSTER_BITS(header) be32toh((header)->cluster_bits) +#define HEADER_CLUSTER_SIZE(header) (1ULL << HEADER_CLUSTER_BITS(header)) +#define HEADER_L2_BITS(header) (HEADER_CLUSTER_BITS(header) - 3) +#define HEADER_SIZE(header) be64toh((header)->size) +#define HEADER_CRYPT_METHOD(header) be32toh((header)->crypt_method) +#define HEADER_L1_SIZE(header) be32toh((header)->l1_size) +#define HEADER_L2_SIZE(header) (HEADER_CLUSTER_SIZE(header)/sizeof(uint64_t)) +#define HEADER_L1_TABLE_OFFSET(header) be64toh((header)->l1_table_offset) + +static uint32_t HEADER_HEADER_LENGTH(const Header *h) { + if (HEADER_VERSION(h) < 3) + return offsetof(Header, incompatible_features); + + return be32toh(h->header_length); +} + +static int copy_cluster( + int sfd, uint64_t soffset, + int dfd, uint64_t doffset, + uint64_t cluster_size, + void *buffer) { + + ssize_t l; + int r; + + r = btrfs_clone_range(sfd, soffset, dfd, doffset, cluster_size); + if (r >= 0) + return r; + + l = pread(sfd, buffer, cluster_size, soffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + l = pwrite(dfd, buffer, cluster_size, doffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + return 0; +} + +static int decompress_cluster( + int sfd, uint64_t soffset, + int dfd, uint64_t doffset, + uint64_t compressed_size, + uint64_t cluster_size, + void *buffer1, + void *buffer2) { + + _cleanup_free_ void *large_buffer = NULL; + z_stream s = {}; + uint64_t sz; + ssize_t l; + int r; + + if (compressed_size > cluster_size) { + /* The usual cluster buffer doesn't suffice, let's + * allocate a larger one, temporarily */ + + large_buffer = malloc(compressed_size); + if (!large_buffer) + return -ENOMEM; + + buffer1 = large_buffer; + } + + l = pread(sfd, buffer1, compressed_size, soffset); + if (l < 0) + return -errno; + if ((uint64_t) l != compressed_size) + return -EIO; + + s.next_in = buffer1; + s.avail_in = compressed_size; + s.next_out = buffer2; + s.avail_out = cluster_size; + + r = inflateInit2(&s, -12); + if (r != Z_OK) + return -EIO; + + r = inflate(&s, Z_FINISH); + sz = (uint8_t*) s.next_out - (uint8_t*) buffer2; + inflateEnd(&s); + if (r != Z_STREAM_END || sz != cluster_size) + return -EIO; + + l = pwrite(dfd, buffer2, cluster_size, doffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + return 0; +} + +static int normalize_offset( + const Header *header, + uint64_t p, + uint64_t *ret, + bool *compressed, + uint64_t *compressed_size) { + + uint64_t q; + + q = be64toh(p); + + if (q & QCOW2_COMPRESSED) { + uint64_t sz, csize_shift, csize_mask; + + if (!compressed) + return -EOPNOTSUPP; + + csize_shift = 64 - 2 - (HEADER_CLUSTER_BITS(header) - 8); + csize_mask = (1ULL << (HEADER_CLUSTER_BITS(header) - 8)) - 1; + sz = (((q >> csize_shift) & csize_mask) + 1) * 512 - (q & 511); + q &= ((1ULL << csize_shift) - 1); + + if (compressed_size) + *compressed_size = sz; + + *compressed = true; + + } else { + if (compressed) { + *compressed = false; + *compressed_size = 0; + } + + if (q & QCOW2_ZERO) { + /* We make no distinction between zero blocks and holes */ + *ret = 0; + return 0; + } + + q &= ~QCOW2_COPIED; + } + + *ret = q; + return q > 0; /* returns positive if not a hole */ +} + +static int verify_header(const Header *header) { + assert(header); + + if (HEADER_MAGIC(header) != QCOW2_MAGIC) + return -EBADMSG; + + if (!IN_SET(HEADER_VERSION(header), 2, 3)) + return -EOPNOTSUPP; + + if (HEADER_CRYPT_METHOD(header) != 0) + return -EOPNOTSUPP; + + if (HEADER_CLUSTER_BITS(header) < 9) /* 512K */ + return -EBADMSG; + + if (HEADER_CLUSTER_BITS(header) > 21) /* 2MB */ + return -EBADMSG; + + if (HEADER_SIZE(header) % HEADER_CLUSTER_SIZE(header) != 0) + return -EBADMSG; + + if (HEADER_L1_SIZE(header) > 32*1024*1024) /* 32MB */ + return -EBADMSG; + + if (HEADER_VERSION(header) == 3) { + + if (header->incompatible_features != 0) + return -EOPNOTSUPP; + + if (HEADER_HEADER_LENGTH(header) < sizeof(Header)) + return -EBADMSG; + } + + return 0; +} + +int qcow2_convert(int qcow2_fd, int raw_fd) { + _cleanup_free_ void *buffer1 = NULL, *buffer2 = NULL; + _cleanup_free_ be64_t *l1_table = NULL, *l2_table = NULL; + uint64_t sz, i; + Header header; + ssize_t l; + int r; + + l = pread(qcow2_fd, &header, sizeof(header), 0); + if (l < 0) + return -errno; + if (l != sizeof(header)) + return -EIO; + + r = verify_header(&header); + if (r < 0) + return r; + + l1_table = new(be64_t, HEADER_L1_SIZE(&header)); + if (!l1_table) + return -ENOMEM; + + l2_table = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!l2_table) + return -ENOMEM; + + buffer1 = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!buffer1) + return -ENOMEM; + + buffer2 = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!buffer2) + return -ENOMEM; + + /* Empty the file if it exists, we rely on zero bits */ + if (ftruncate(raw_fd, 0) < 0) + return -errno; + + if (ftruncate(raw_fd, HEADER_SIZE(&header)) < 0) + return -errno; + + sz = sizeof(uint64_t) * HEADER_L1_SIZE(&header); + l = pread(qcow2_fd, l1_table, sz, HEADER_L1_TABLE_OFFSET(&header)); + if (l < 0) + return -errno; + if ((uint64_t) l != sz) + return -EIO; + + for (i = 0; i < HEADER_L1_SIZE(&header); i ++) { + uint64_t l2_begin, j; + + r = normalize_offset(&header, l1_table[i], &l2_begin, NULL, NULL); + if (r < 0) + return r; + if (r == 0) + continue; + + l = pread(qcow2_fd, l2_table, HEADER_CLUSTER_SIZE(&header), l2_begin); + if (l < 0) + return -errno; + if ((uint64_t) l != HEADER_CLUSTER_SIZE(&header)) + return -EIO; + + for (j = 0; j < HEADER_L2_SIZE(&header); j++) { + uint64_t data_begin, p, compressed_size; + bool compressed; + + p = ((i << HEADER_L2_BITS(&header)) + j) << HEADER_CLUSTER_BITS(&header); + + r = normalize_offset(&header, l2_table[j], &data_begin, &compressed, &compressed_size); + if (r < 0) + return r; + if (r == 0) + continue; + + if (compressed) + r = decompress_cluster( + qcow2_fd, data_begin, + raw_fd, p, + compressed_size, HEADER_CLUSTER_SIZE(&header), + buffer1, buffer2); + else + r = copy_cluster( + qcow2_fd, data_begin, + raw_fd, p, + HEADER_CLUSTER_SIZE(&header), buffer1); + if (r < 0) + return r; + } + } + + return 0; +} + +int qcow2_detect(int fd) { + be32_t id; + ssize_t l; + + l = pread(fd, &id, sizeof(id), 0); + if (l < 0) + return -errno; + if (l != sizeof(id)) + return -EIO; + + return htobe32(QCOW2_MAGIC) == id; +} diff --git a/src/import/qcow2-util.h b/src/import/qcow2-util.h new file mode 100644 index 0000000..f17c159 --- /dev/null +++ b/src/import/qcow2-util.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +int qcow2_detect(int fd); +int qcow2_convert(int qcow2_fd, int raw_fd); diff --git a/src/import/test-qcow2.c b/src/import/test-qcow2.c new file mode 100644 index 0000000..77fed01 --- /dev/null +++ b/src/import/test-qcow2.c @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "fd-util.h" +#include "log.h" +#include "qcow2-util.h" + +int main(int argc, char *argv[]) { + _cleanup_close_ int sfd = -1, dfd = -1; + int r; + + if (argc != 3) { + log_error("Needs two arguments."); + return EXIT_FAILURE; + } + + sfd = open(argv[1], O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (sfd < 0) { + log_error_errno(errno, "Can't open source file: %m"); + return EXIT_FAILURE; + } + + dfd = open(argv[2], O_WRONLY|O_CREAT|O_CLOEXEC|O_NOCTTY, 0666); + if (dfd < 0) { + log_error_errno(errno, "Can't open destination file: %m"); + return EXIT_FAILURE; + } + + r = qcow2_convert(sfd, dfd); + if (r < 0) { + log_error_errno(r, "Failed to unpack: %m"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} |