/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- * vi:set noexpandtab tabstop=8 shiftwidth=8: * * Copyright (C) 2018 Endless Mobile, Inc. * * Authors: Joaquim Rocha * * SPDX-License-Identifier: GPL-2.0-or-later */ /* * SECTION:gs-external-appstream-urls * @short_description: Provides support for downloading external AppStream files. * * This downloads the set of configured external AppStream files, and caches * them locally. * * According to the `external-appstream-system-wide` GSetting, the files will * either be downloaded to a per-user cache, or to a system-wide cache. In the * case of a system-wide cache, they are downloaded to a temporary file writable * by the user, and then the suexec binary `gnome-software-install-appstream` is * run to copy them to the system location. * * All the downloads are done in the default #GMainContext for the thread which * calls gs_external_appstream_refresh_async(). They are done in parallel and * the async refresh function will only complete once the last download is * complete. * * Progress data is reported via a callback, and gives the total progress of all * parallel downloads. Internally this is done by updating #ProgressTuple * structs as each download progresses. A periodic timeout callback sums these * and reports the total progress to the caller. That means that progress * reports from gs_external_appstream_refresh_async() are done at a constant * frequency. * * To test this code locally you will probably want to change your GSettings * configuration to add some external AppStream URIs: * ``` * gsettings set org.gnome.software external-appstream-urls '["https://example.com/appdata.xml.gz"]' * ``` * * When you are done with development, run the following command to use the real * external AppStream list again: * ``` * gsettings reset org.gnome.software external-appstream-urls * ``` * * Since: 42 */ #include "config.h" #include #include #include #include #include #include "gs-external-appstream-utils.h" #define APPSTREAM_SYSTEM_DIR LOCALSTATEDIR "/cache/swcatalog/xml" G_DEFINE_QUARK (gs-external-appstream-error-quark, gs_external_appstream_error) gchar * gs_external_appstream_utils_get_file_cache_path (const gchar *file_name) { g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s", file_name); return g_build_filename (APPSTREAM_SYSTEM_DIR, prefixed_file_name, NULL); } /* To be able to delete old files, when the path changed */ gchar * gs_external_appstream_utils_get_legacy_file_cache_path (const gchar *file_name) { g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s", file_name); return g_build_filename (LOCALSTATEDIR "/cache/app-info/xmls", prefixed_file_name, NULL); } const gchar * gs_external_appstream_utils_get_system_dir (void) { return APPSTREAM_SYSTEM_DIR; } static gboolean gs_external_appstream_check (GFile *appstream_file, guint64 cache_age_secs) { guint64 appstream_file_age = gs_utils_get_file_age (appstream_file); return appstream_file_age >= cache_age_secs; } static gboolean gs_external_appstream_install (const gchar *appstream_file, GCancellable *cancellable, GError **error) { g_autoptr(GSubprocess) subprocess = NULL; const gchar *argv[] = { "pkexec", LIBEXECDIR "/gnome-software-install-appstream", appstream_file, NULL}; g_debug ("Installing the appstream file %s in the system", appstream_file); subprocess = g_subprocess_newv (argv, G_SUBPROCESS_FLAGS_STDOUT_PIPE | G_SUBPROCESS_FLAGS_STDIN_PIPE, error); if (subprocess == NULL) return FALSE; return g_subprocess_wait_check (subprocess, cancellable, error); } static void download_replace_file_cb (GObject *source_object, GAsyncResult *result, gpointer user_data); static void download_stream_cb (GObject *source_object, GAsyncResult *result, gpointer user_data); /* A tuple to store the last-received progress data for a single download. * Each download (refresh_url_async()) has a pointer to the relevant * #ProgressTuple for its download. These are stored in an array in #RefreshData * and a timeout callback periodically sums them all and reports progress to the * caller. */ typedef struct { gsize bytes_downloaded; gsize total_download_size; } ProgressTuple; typedef struct { /* Input data. */ gchar *url; /* (not nullable) (owned) */ GTask *task; /* (not nullable) (owned) */ GFile *output_file; /* (not nullable) (owned) */ ProgressTuple *progress_tuple; /* (not nullable) */ SoupSession *soup_session; /* (not nullable) (owned) */ gboolean system_wide; /* In-progress data. */ gchar *last_etag; /* (nullable) (owned) */ GDateTime *last_modified_date; /* (nullable) (owned) */ } DownloadAppStreamData; static void download_appstream_data_free (DownloadAppStreamData *data) { g_free (data->url); g_clear_object (&data->task); g_clear_object (&data->output_file); g_clear_object (&data->soup_session); g_free (data->last_etag); g_clear_pointer (&data->last_modified_date, g_date_time_unref); g_free (data); } G_DEFINE_AUTOPTR_CLEANUP_FUNC (DownloadAppStreamData, download_appstream_data_free) static void refresh_url_progress_cb (gsize bytes_downloaded, gsize total_download_size, gpointer user_data) { ProgressTuple *tuple = user_data; tuple->bytes_downloaded = bytes_downloaded; tuple->total_download_size = total_download_size; /* The timeout callback in progress_cb() periodically sums these. No * need to notify of progress from here. */ } static void refresh_url_async (GSettings *settings, const gchar *cache_kind, const gchar *url, SoupSession *soup_session, guint64 cache_age_secs, ProgressTuple *progress_tuple, gchar **out_appstream_path, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { g_autoptr(GTask) task = NULL; g_autofree gchar *basename = NULL; g_autofree gchar *basename_url = g_path_get_basename (url); /* make sure different uris with same basenames differ */ g_autofree gchar *hash = NULL; g_autofree gchar *target_file_path = NULL; g_autoptr(GFile) target_file = NULL; g_autoptr(GFile) tmp_file_parent = NULL; g_autoptr(GFile) tmp_file = NULL; g_autoptr(GsApp) app_dl = gs_app_new ("external-appstream"); g_autoptr(GError) local_error = NULL; DownloadAppStreamData *data; gboolean system_wide; task = g_task_new (NULL, cancellable, callback, user_data); g_task_set_source_tag (task, refresh_url_async); /* Calculate the basename of the target file. */ hash = g_compute_checksum_for_string (G_CHECKSUM_SHA1, url, -1); if (hash == NULL) { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING, "Failed to hash URI ‘%s’", url); return; } basename = g_strdup_printf ("%s-%s", hash, basename_url); /* Are we downloading for a given cache kind, the user, or the system? */ system_wide = cache_kind == NULL && g_settings_get_boolean (settings, "external-appstream-system-wide"); /* Check cache file age. */ if (cache_kind != NULL) { target_file_path = gs_utils_get_cache_filename (cache_kind, basename, GS_UTILS_CACHE_FLAG_WRITEABLE | GS_UTILS_CACHE_FLAG_CREATE_DIRECTORY, &local_error); if (target_file_path == NULL) { g_task_return_error (task, g_steal_pointer (&local_error)); return; } } else if (system_wide) { target_file_path = gs_external_appstream_utils_get_file_cache_path (basename); } else { g_autofree gchar *legacy_file_path = NULL; target_file_path = g_build_filename (g_get_user_data_dir (), "swcatalog", "xml", basename, NULL); /* Delete an old file, from a legacy location */ legacy_file_path = g_build_filename (g_get_user_data_dir (), "app-info", "xmls", basename, NULL); if (g_unlink (legacy_file_path) == -1) { int errn = errno; if (errn != ENOENT) g_debug ("Failed to unlink '%s': %s", legacy_file_path, g_strerror (errn)); } } target_file = g_file_new_for_path (target_file_path); if (!gs_external_appstream_check (target_file, cache_age_secs)) { g_debug ("skipping updating external appstream file %s: " "cache age is older than file", target_file_path); if (out_appstream_path != NULL) { *out_appstream_path = g_steal_pointer (&target_file_path); } g_task_return_boolean (task, TRUE); return; } if (out_appstream_path != NULL) { *out_appstream_path = g_steal_pointer (&target_file_path); } /* If downloading system wide, write the download contents into a * temporary file that will be copied into the system location later. */ if (system_wide) { g_autofree gchar *tmp_file_path = NULL; tmp_file_path = gs_utils_get_cache_filename ("external-appstream", basename, GS_UTILS_CACHE_FLAG_WRITEABLE | GS_UTILS_CACHE_FLAG_CREATE_DIRECTORY, &local_error); if (tmp_file_path == NULL) { g_task_return_error (task, g_steal_pointer (&local_error)); return; } tmp_file = g_file_new_for_path (tmp_file_path); } else { tmp_file = g_object_ref (target_file); } gs_app_set_summary_missing (app_dl, /* TRANSLATORS: status text when downloading */ _("Downloading extra metadata files…")); data = g_new0 (DownloadAppStreamData, 1); data->url = g_strdup (url); data->task = g_object_ref (task); data->output_file = g_object_ref (tmp_file); data->progress_tuple = progress_tuple; data->soup_session = g_object_ref (soup_session); data->system_wide = system_wide; g_task_set_task_data (task, data, (GDestroyNotify) download_appstream_data_free); /* Create the destination file’s directory. * FIXME: This should be made async; it hasn’t done for now as it’s * likely to be fast. */ tmp_file_parent = g_file_get_parent (tmp_file); if (tmp_file_parent != NULL && !g_file_make_directory_with_parents (tmp_file_parent, cancellable, &local_error) && !g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_EXISTS)) { g_task_return_error (task, g_steal_pointer (&local_error)); return; } g_clear_error (&local_error); /* Query the ETag and modification date of the target file, if the file already exists. For * system-wide installations, this is the ETag of the AppStream file installed system-wide. * For local installations, this is just the local output file. */ data->last_etag = gs_utils_get_file_etag (target_file, &data->last_modified_date, cancellable); g_debug ("Queried ETag of file %s: %s", g_file_peek_path (target_file), data->last_etag); /* Create the output file */ g_file_replace_async (tmp_file, NULL, /* ETag */ FALSE, /* make_backup */ G_FILE_CREATE_PRIVATE | G_FILE_CREATE_REPLACE_DESTINATION, G_PRIORITY_LOW, cancellable, download_replace_file_cb, g_steal_pointer (&task)); } static void download_replace_file_cb (GObject *source_object, GAsyncResult *result, gpointer user_data) { GFile *output_file = G_FILE (source_object); g_autoptr(GTask) task = g_steal_pointer (&user_data); GCancellable *cancellable = g_task_get_cancellable (task); DownloadAppStreamData *data = g_task_get_task_data (task); g_autoptr(GFileOutputStream) output_stream = NULL; g_autoptr(GError) local_error = NULL; output_stream = g_file_replace_finish (output_file, result, &local_error); if (output_stream == NULL) { g_task_return_error (task, g_steal_pointer (&local_error)); return; } /* Do the download. */ gs_download_stream_async (data->soup_session, data->url, G_OUTPUT_STREAM (output_stream), data->last_etag, data->last_modified_date, G_PRIORITY_LOW, refresh_url_progress_cb, data->progress_tuple, cancellable, download_stream_cb, g_steal_pointer (&task)); } static void download_stream_cb (GObject *source_object, GAsyncResult *result, gpointer user_data) { SoupSession *soup_session = SOUP_SESSION (source_object); g_autoptr(GTask) task = g_steal_pointer (&user_data); GCancellable *cancellable = g_task_get_cancellable (task); DownloadAppStreamData *data = g_task_get_task_data (task); g_autoptr(GError) local_error = NULL; g_autofree gchar *new_etag = NULL; if (!gs_download_stream_finish (soup_session, result, &new_etag, NULL, &local_error)) { if (data->system_wide && g_error_matches (local_error, GS_DOWNLOAD_ERROR, GS_DOWNLOAD_ERROR_NOT_MODIFIED)) { g_debug ("External AppStream file not modified, removing temporary download file %s", g_file_peek_path (data->output_file)); /* System-wide installs should delete the empty file created when preparing to * download the external AppStream file. */ g_file_delete_async (data->output_file, G_PRIORITY_LOW, NULL, NULL, NULL); g_task_return_boolean (task, TRUE); } else if (!g_network_monitor_get_network_available (g_network_monitor_get_default ())) { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_NO_NETWORK, "External AppStream could not be downloaded due to being offline"); } else { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING, "Server returned no data for external AppStream file: %s", local_error->message); } return; } g_debug ("Downloaded appstream file %s", g_file_peek_path (data->output_file)); gs_utils_set_file_etag (data->output_file, new_etag, cancellable); if (data->system_wide) { /* install file systemwide */ if (!gs_external_appstream_install (g_file_peek_path (data->output_file), cancellable, &local_error)) { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_INSTALLING_ON_SYSTEM, "Error installing external AppStream file on system: %s", local_error->message); return; } g_debug ("Installed appstream file %s", g_file_peek_path (data->output_file)); } g_task_return_boolean (task, TRUE); } static gboolean refresh_url_finish (GAsyncResult *result, GError **error) { return g_task_propagate_boolean (G_TASK (result), error); } static void refresh_cb (GObject *source_object, GAsyncResult *result, gpointer user_data); static gboolean progress_cb (gpointer user_data); static void finish_refresh_op (GTask *task, GError *error); typedef struct { /* Input data. */ guint64 cache_age_secs; /* In-progress data. */ guint n_pending_ops; GError *error; /* (nullable) (owned) */ GsDownloadProgressCallback progress_callback; /* (nullable) */ gpointer progress_user_data; /* (closure progress_callback) */ gsize n_appstream_urls; ProgressTuple *progress_tuples; /* (array length=n_appstream_urls) (owned) */ GSource *progress_source; /* (owned) */ /* This is a fixed-sized array that contains (n_appstream_urls + 1) * items, the last one being guaranteed to be NULL. It is used like a * fixed-sized array internally, but it turned into a NULL-terminated * array into gs_external_appstream_refresh_finish() to avoid * reallocation. */ gchar **appstream_paths; /* (array length=n_appstream_urls) (owned) */ } RefreshData; static void refresh_data_free (RefreshData *data) { g_assert (data->n_pending_ops == 0); /* If this was set it should have been stolen for g_task_return_error() * by now. */ g_assert (data->error == NULL); /* Similarly, progress reporting should have been stopped by now. */ g_assert (g_source_is_destroyed (data->progress_source)); g_source_unref (data->progress_source); g_free (data->progress_tuples); /* This doesn’t use g_strfreev() because it is a fixed-sized array, any * element of data->appstream_paths may be NULL. It itself can be NULL * if it has been stolen in gs_external_appstream_refresh_finish(). */ if (data->appstream_paths != NULL) { for (gsize i = 0; i < data->n_appstream_urls; i++) { g_clear_pointer (&data->appstream_paths[i], g_free); } g_clear_pointer (&data->appstream_paths, g_free); } g_free (data); } G_DEFINE_AUTOPTR_CLEANUP_FUNC (RefreshData, refresh_data_free) /** * gs_external_appstream_refresh_async: * @cache_kind: (nullable): a cache kind, e.g. "fwupd" or "screenshots/123x456", or %NULL * @appstream_urls: a %NULL-terminated array of URLs * @cache_age_secs: cache age, in seconds, as passed to #GsPluginClass.refresh_metadata_async() * @progress_callback: (nullable): callback to call with progress information * @progress_user_data: (nullable) (closure progress_callback): data to pass * to @progress_callback * @cancellable: (nullable): a #GCancellable, or %NULL * @callback: function call when the asynchronous operation is complete * @user_data: data to pass to @callback * * Refresh any external appstream files, if the cache is too old. * * If @cache_kind is set, the files will be cached into a per-user cache * directory, and into a global cache othwerwise. The global directory will be * system-wide or user-specific according to the * `external-appstream-system-wide` setting. * * If a plugin requests a file to be saved in the cache it is the plugins * responsibility to remove the file when it is no longer valid or is too old * -- gnome-software will not ever clean the cache for the plugin. * For this reason it is a good idea to use the plugin name as @cache_kind. * * Since: 48 */ void gs_external_appstream_refresh_async (const gchar *cache_kind, GStrv appstream_urls, guint64 cache_age_secs, GsDownloadProgressCallback progress_callback, gpointer progress_user_data, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { g_autoptr(GSettings) settings = NULL; gsize n_appstream_urls; g_autoptr(SoupSession) soup_session = NULL; g_autoptr(GTask) task = NULL; RefreshData *data; g_autoptr(RefreshData) data_owned = NULL; /* Chosen to allow a few UI updates per second without updating the * progress label so often it’s unreadable. */ const guint progress_update_period_ms = 300; task = g_task_new (NULL, cancellable, callback, user_data); g_task_set_source_tag (task, gs_external_appstream_refresh_async); settings = g_settings_new ("org.gnome.software"); soup_session = gs_build_soup_session (); n_appstream_urls = g_strv_length (appstream_urls); data = data_owned = g_new0 (RefreshData, 1); data->progress_callback = progress_callback; data->progress_user_data = progress_user_data; data->n_appstream_urls = n_appstream_urls; data->progress_tuples = g_new0 (ProgressTuple, n_appstream_urls); data->progress_source = g_timeout_source_new (progress_update_period_ms); /* We want to use it as a fixed-size array internally but to return it * as a NULL-terminated array, so we have to add an extra terminating * item at the end. */ data->appstream_paths = g_new0 (gchar *, n_appstream_urls + 1); g_task_set_task_data (task, g_steal_pointer (&data_owned), (GDestroyNotify) refresh_data_free); /* Set up the progress timeout. This periodically sums up the progress * tuples in `data->progress_tuples` and reports them to the calling * function via @progress_callback, giving an overall progress for all * the parallel operations. */ g_source_set_callback (data->progress_source, progress_cb, g_object_ref (task), g_object_unref); g_source_attach (data->progress_source, g_main_context_get_thread_default ()); /* Refresh all the URIs in parallel. */ data->n_pending_ops = 1; for (gsize i = 0; i < n_appstream_urls; i++) { /* localhost is safe to communicate with in an unencrypted way. * It is unlikely to be used in real life scenarios, but it's * used in some tests. We could use TLS in the tests, but it * would needlessly complexify them. */ if (!g_str_has_prefix (appstream_urls[i], "https:") && !g_str_has_prefix (appstream_urls[i], "http://localhost/") && !g_str_has_prefix (appstream_urls[i], "http://localhost:")) { g_warning ("Not considering %s as an external " "appstream source: please use an https URL", appstream_urls[i]); continue; } data->n_pending_ops++; refresh_url_async (settings, cache_kind, appstream_urls[i], soup_session, cache_age_secs, &data->progress_tuples[i], &data->appstream_paths[i], cancellable, refresh_cb, g_object_ref (task)); } finish_refresh_op (task, NULL); } static void refresh_cb (GObject *source_object, GAsyncResult *result, gpointer user_data) { g_autoptr(GTask) task = g_steal_pointer (&user_data); g_autoptr(GError) local_error = NULL; refresh_url_finish (result, &local_error); finish_refresh_op (task, g_steal_pointer (&local_error)); } static gboolean progress_cb (gpointer user_data) { GTask *task = G_TASK (user_data); RefreshData *data = g_task_get_task_data (task); gsize parallel_bytes_downloaded = 0, parallel_total_download_size = 0; /* Sum up the progress numerator and denominator for all parallel * downloads. */ for (gsize i = 0; i < data->n_appstream_urls; i++) { const ProgressTuple *progress_tuple = &data->progress_tuples[i]; if (!g_size_checked_add (¶llel_bytes_downloaded, parallel_bytes_downloaded, progress_tuple->bytes_downloaded)) parallel_bytes_downloaded = G_MAXSIZE; if (!g_size_checked_add (¶llel_total_download_size, parallel_total_download_size, progress_tuple->total_download_size)) parallel_total_download_size = G_MAXSIZE; } /* Report progress to the calling function. */ if (data->progress_callback != NULL) data->progress_callback (parallel_bytes_downloaded, parallel_total_download_size, data->progress_user_data); return G_SOURCE_CONTINUE; } /* @error is (transfer full) if non-%NULL */ static void finish_refresh_op (GTask *task, GError *error) { RefreshData *data = g_task_get_task_data (task); g_autoptr(GError) error_owned = g_steal_pointer (&error); if (data->error == NULL && error_owned != NULL) data->error = g_steal_pointer (&error_owned); else if (error_owned != NULL) g_debug ("Additional error while refreshing external appstream: %s", error_owned->message); g_assert (data->n_pending_ops > 0); data->n_pending_ops--; if (data->n_pending_ops > 0) return; /* Emit one final progress update, then stop any further ones. */ progress_cb (task); g_source_destroy (data->progress_source); /* All complete. */ if (data->error != NULL) g_task_return_error (task, g_steal_pointer (&data->error)); else g_task_return_boolean (task, TRUE); } /** * gs_external_appstream_refresh_finish: * @result: a #GAsyncResult * @out_appstream_paths: (out) (transfer full) (optional) (nullable): return * location for the %NULL-terminated array of downloaded appstream file paths, * or %NULL to ignore * @error: return location for a #GError, or %NULL * * Finish an asynchronous refresh operation started with * gs_external_appstream_refresh_async(). * * Returns: %TRUE on success, %FALSE otherwise * Since: 48 */ gboolean gs_external_appstream_refresh_finish (GAsyncResult *result, gchar ***out_appstream_paths, GError **error) { GTask *task; RefreshData *data; g_auto(GStrv) appstream_paths_tmp = NULL; gboolean success; g_return_val_if_fail (g_task_is_valid (result, NULL), FALSE); g_return_val_if_fail (error == NULL || *error == NULL, FALSE); task = G_TASK (result); data = g_task_get_task_data (task); if (out_appstream_paths != NULL) { /* Turn the paths array from a fixed-size array into a * NULL-terminated one, so we can return it without copying it. */ for (gsize i = 0, j = 0; i < data->n_appstream_urls; i++) { if (data->appstream_paths[i] == NULL) { continue; } if (i != j) { data->appstream_paths[j] = g_steal_pointer (&data->appstream_paths[i]); } j++; } appstream_paths_tmp = g_steal_pointer (&data->appstream_paths); } success = g_task_propagate_boolean (G_TASK (result), error); if (success && out_appstream_paths != NULL) { *out_appstream_paths = g_steal_pointer (&appstream_paths_tmp); } return success; }