/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- * vi:set noexpandtab tabstop=8 shiftwidth=8: * * Copyright (C) 2018 Endless Mobile, Inc. * * Authors: Joaquim Rocha * * SPDX-License-Identifier: GPL-2.0+ */ /* * SECTION:gs-external-appstream-urls * @short_description: Provides support for downloading external AppStream files. * * This downloads the set of configured external AppStream files, and caches * them locally. * * According to the `external-appstream-system-wide` GSetting, the files will * either be downloaded to a per-user cache, or to a system-wide cache. In the * case of a system-wide cache, they are downloaded to a temporary file writable * by the user, and then the suexec binary `gnome-software-install-appstream` is * run to copy them to the system location. * * All the downloads are done in the default #GMainContext for the thread which * calls gs_external_appstream_refresh_async(). They are done in parallel and * the async refresh function will only complete once the last download is * complete. * * Progress data is reported via a callback, and gives the total progress of all * parallel downloads. Internally this is done by updating #ProgressTuple * structs as each download progresses. A periodic timeout callback sums these * and reports the total progress to the caller. That means that progress * reports from gs_external_appstream_refresh_async() are done at a constant * frequency. * * To test this code locally you will probably want to change your GSettings * configuration to add some external AppStream URIs: * ``` * gsettings set org.gnome.software external-appstream-urls '["https://example.com/appdata.xml.gz"]' * ``` * * When you are done with development, run the following command to use the real * external AppStream list again: * ``` * gsettings reset org.gnome.software external-appstream-urls * ``` * * Since: 42 */ #include #include #include #include #include #include "gs-external-appstream-utils.h" #define APPSTREAM_SYSTEM_DIR LOCALSTATEDIR "/cache/swcatalog/xml" G_DEFINE_QUARK (gs-external-appstream-error-quark, gs_external_appstream_error) gchar * gs_external_appstream_utils_get_file_cache_path (const gchar *file_name) { g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s", file_name); return g_build_filename (APPSTREAM_SYSTEM_DIR, prefixed_file_name, NULL); } /* To be able to delete old files, when the path changed */ gchar * gs_external_appstream_utils_get_legacy_file_cache_path (const gchar *file_name) { g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s", file_name); return g_build_filename (LOCALSTATEDIR "/cache/app-info/xmls", prefixed_file_name, NULL); } const gchar * gs_external_appstream_utils_get_system_dir (void) { return APPSTREAM_SYSTEM_DIR; } static gboolean gs_external_appstream_check (GFile *appstream_file, guint64 cache_age_secs) { guint64 appstream_file_age = gs_utils_get_file_age (appstream_file); return appstream_file_age >= cache_age_secs; } static gboolean gs_external_appstream_install (const gchar *appstream_file, GCancellable *cancellable, GError **error) { g_autoptr(GSubprocess) subprocess = NULL; const gchar *argv[] = { "pkexec", LIBEXECDIR "/gnome-software-install-appstream", appstream_file, NULL}; g_debug ("Installing the appstream file %s in the system", appstream_file); subprocess = g_subprocess_newv (argv, G_SUBPROCESS_FLAGS_STDOUT_PIPE | G_SUBPROCESS_FLAGS_STDIN_PIPE, error); if (subprocess == NULL) return FALSE; return g_subprocess_wait_check (subprocess, cancellable, error); } static void download_replace_file_cb (GObject *source_object, GAsyncResult *result, gpointer user_data); static void download_stream_cb (GObject *source_object, GAsyncResult *result, gpointer user_data); /* A tuple to store the last-received progress data for a single download. * Each download (refresh_url_async()) has a pointer to the relevant * #ProgressTuple for its download. These are stored in an array in #RefreshData * and a timeout callback periodically sums them all and reports progress to the * caller. */ typedef struct { gsize bytes_downloaded; gsize total_download_size; } ProgressTuple; typedef struct { /* Input data. */ gchar *url; /* (not nullable) (owned) */ GTask *task; /* (not nullable) (owned) */ GFile *output_file; /* (not nullable) (owned) */ ProgressTuple *progress_tuple; /* (not nullable) */ SoupSession *soup_session; /* (not nullable) (owned) */ gboolean system_wide; /* In-progress data. */ gchar *last_etag; /* (nullable) (owned) */ GDateTime *last_modified_date; /* (nullable) (owned) */ } DownloadAppStreamData; static void download_appstream_data_free (DownloadAppStreamData *data) { g_free (data->url); g_clear_object (&data->task); g_clear_object (&data->output_file); g_clear_object (&data->soup_session); g_free (data->last_etag); g_clear_pointer (&data->last_modified_date, g_date_time_unref); g_free (data); } G_DEFINE_AUTOPTR_CLEANUP_FUNC (DownloadAppStreamData, download_appstream_data_free) static void refresh_url_progress_cb (gsize bytes_downloaded, gsize total_download_size, gpointer user_data) { ProgressTuple *tuple = user_data; tuple->bytes_downloaded = bytes_downloaded; tuple->total_download_size = total_download_size; /* The timeout callback in progress_cb() periodically sums these. No * need to notify of progress from here. */ } static void refresh_url_async (GSettings *settings, const gchar *url, SoupSession *soup_session, guint64 cache_age_secs, ProgressTuple *progress_tuple, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { g_autoptr(GTask) task = NULL; g_autofree gchar *basename = NULL; g_autofree gchar *basename_url = g_path_get_basename (url); /* make sure different uris with same basenames differ */ g_autofree gchar *hash = NULL; g_autofree gchar *target_file_path = NULL; g_autoptr(GFile) target_file = NULL; g_autoptr(GFile) tmp_file_parent = NULL; g_autoptr(GFile) tmp_file = NULL; g_autoptr(GsApp) app_dl = gs_app_new ("external-appstream"); g_autoptr(GError) local_error = NULL; DownloadAppStreamData *data; gboolean system_wide; task = g_task_new (NULL, cancellable, callback, user_data); g_task_set_source_tag (task, refresh_url_async); /* Calculate the basename of the target file. */ hash = g_compute_checksum_for_string (G_CHECKSUM_SHA1, url, -1); if (hash == NULL) { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING, "Failed to hash URI ‘%s’", url); return; } basename = g_strdup_printf ("%s-%s", hash, basename_url); /* Are we downloading for the user, or the system? */ system_wide = g_settings_get_boolean (settings, "external-appstream-system-wide"); /* Check cache file age. */ if (system_wide) { target_file_path = gs_external_appstream_utils_get_file_cache_path (basename); } else { g_autofree gchar *legacy_file_path = NULL; target_file_path = g_build_filename (g_get_user_data_dir (), "swcatalog", "xml", basename, NULL); /* Delete an old file, from a legacy location */ legacy_file_path = g_build_filename (g_get_user_data_dir (), "app-info", "xmls", basename, NULL); if (g_unlink (legacy_file_path) == -1) { int errn = errno; if (errn != ENOENT) g_debug ("Failed to unlink '%s': %s", legacy_file_path, g_strerror (errn)); } } target_file = g_file_new_for_path (target_file_path); if (!gs_external_appstream_check (target_file, cache_age_secs)) { g_debug ("skipping updating external appstream file %s: " "cache age is older than file", target_file_path); g_task_return_boolean (task, TRUE); return; } /* If downloading system wide, write the download contents into a * temporary file that will be copied into the system location later. */ if (system_wide) { g_autofree gchar *tmp_file_path = NULL; tmp_file_path = gs_utils_get_cache_filename ("external-appstream", basename, GS_UTILS_CACHE_FLAG_WRITEABLE | GS_UTILS_CACHE_FLAG_CREATE_DIRECTORY, &local_error); if (tmp_file_path == NULL) { g_task_return_error (task, g_steal_pointer (&local_error)); return; } tmp_file = g_file_new_for_path (tmp_file_path); } else { tmp_file = g_object_ref (target_file); } gs_app_set_summary_missing (app_dl, /* TRANSLATORS: status text when downloading */ _("Downloading extra metadata files…")); data = g_new0 (DownloadAppStreamData, 1); data->url = g_strdup (url); data->task = g_object_ref (task); data->output_file = g_object_ref (tmp_file); data->progress_tuple = progress_tuple; data->soup_session = g_object_ref (soup_session); data->system_wide = system_wide; g_task_set_task_data (task, data, (GDestroyNotify) download_appstream_data_free); /* Create the destination file’s directory. * FIXME: This should be made async; it hasn’t done for now as it’s * likely to be fast. */ tmp_file_parent = g_file_get_parent (tmp_file); if (tmp_file_parent != NULL && !g_file_make_directory_with_parents (tmp_file_parent, cancellable, &local_error) && !g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_EXISTS)) { g_task_return_error (task, g_steal_pointer (&local_error)); return; } g_clear_error (&local_error); /* Query the ETag and modification date of the target file, if the file already exists. For * system-wide installations, this is the ETag of the AppStream file installed system-wide. * For local installations, this is just the local output file. */ data->last_etag = gs_utils_get_file_etag (target_file, &data->last_modified_date, cancellable); g_debug ("Queried ETag of file %s: %s", g_file_peek_path (target_file), data->last_etag); /* Create the output file */ g_file_replace_async (tmp_file, NULL, /* ETag */ FALSE, /* make_backup */ G_FILE_CREATE_PRIVATE | G_FILE_CREATE_REPLACE_DESTINATION, G_PRIORITY_LOW, cancellable, download_replace_file_cb, g_steal_pointer (&task)); } static void download_replace_file_cb (GObject *source_object, GAsyncResult *result, gpointer user_data) { GFile *output_file = G_FILE (source_object); g_autoptr(GTask) task = g_steal_pointer (&user_data); GCancellable *cancellable = g_task_get_cancellable (task); DownloadAppStreamData *data = g_task_get_task_data (task); g_autoptr(GFileOutputStream) output_stream = NULL; g_autoptr(GError) local_error = NULL; output_stream = g_file_replace_finish (output_file, result, &local_error); if (output_stream == NULL) { g_task_return_error (task, g_steal_pointer (&local_error)); return; } /* Do the download. */ gs_download_stream_async (data->soup_session, data->url, G_OUTPUT_STREAM (output_stream), data->last_etag, data->last_modified_date, G_PRIORITY_LOW, refresh_url_progress_cb, data->progress_tuple, cancellable, download_stream_cb, g_steal_pointer (&task)); } static void download_stream_cb (GObject *source_object, GAsyncResult *result, gpointer user_data) { SoupSession *soup_session = SOUP_SESSION (source_object); g_autoptr(GTask) task = g_steal_pointer (&user_data); GCancellable *cancellable = g_task_get_cancellable (task); DownloadAppStreamData *data = g_task_get_task_data (task); g_autoptr(GError) local_error = NULL; g_autofree gchar *new_etag = NULL; if (!gs_download_stream_finish (soup_session, result, &new_etag, NULL, &local_error)) { if (data->system_wide && g_error_matches (local_error, GS_DOWNLOAD_ERROR, GS_DOWNLOAD_ERROR_NOT_MODIFIED)) { g_debug ("External AppStream file not modified, removing temporary download file %s", g_file_peek_path (data->output_file)); /* System-wide installs should delete the empty file created when preparing to * download the external AppStream file. */ g_file_delete_async (data->output_file, G_PRIORITY_LOW, NULL, NULL, NULL); g_task_return_boolean (task, TRUE); } else if (!g_network_monitor_get_network_available (g_network_monitor_get_default ())) { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_NO_NETWORK, "External AppStream could not be downloaded due to being offline"); } else { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING, "Server returned no data for external AppStream file: %s", local_error->message); } return; } g_debug ("Downloaded appstream file %s", g_file_peek_path (data->output_file)); gs_utils_set_file_etag (data->output_file, new_etag, cancellable); if (data->system_wide) { /* install file systemwide */ if (!gs_external_appstream_install (g_file_peek_path (data->output_file), cancellable, &local_error)) { g_task_return_new_error (task, GS_EXTERNAL_APPSTREAM_ERROR, GS_EXTERNAL_APPSTREAM_ERROR_INSTALLING_ON_SYSTEM, "Error installing external AppStream file on system: %s", local_error->message); return; } g_debug ("Installed appstream file %s", g_file_peek_path (data->output_file)); } g_task_return_boolean (task, TRUE); } static gboolean refresh_url_finish (GAsyncResult *result, GError **error) { return g_task_propagate_boolean (G_TASK (result), error); } static void refresh_cb (GObject *source_object, GAsyncResult *result, gpointer user_data); static gboolean progress_cb (gpointer user_data); static void finish_refresh_op (GTask *task, GError *error); typedef struct { /* Input data. */ guint64 cache_age_secs; /* In-progress data. */ guint n_pending_ops; GError *error; /* (nullable) (owned) */ gsize n_appstream_urls; GsDownloadProgressCallback progress_callback; /* (nullable) */ gpointer progress_user_data; /* (closure progress_callback) */ ProgressTuple *progress_tuples; /* (array length=n_appstream_urls) (owned) */ GSource *progress_source; /* (owned) */ } RefreshData; static void refresh_data_free (RefreshData *data) { g_assert (data->n_pending_ops == 0); /* If this was set it should have been stolen for g_task_return_error() * by now. */ g_assert (data->error == NULL); /* Similarly, progress reporting should have been stopped by now. */ g_assert (g_source_is_destroyed (data->progress_source)); g_source_unref (data->progress_source); g_free (data->progress_tuples); g_free (data); } G_DEFINE_AUTOPTR_CLEANUP_FUNC (RefreshData, refresh_data_free) /** * gs_external_appstream_refresh_async: * @cache_age_secs: cache age, in seconds, as passed to #GsPluginClass.refresh_metadata_async() * @progress_callback: (nullable): callback to call with progress information * @progress_user_data: (nullable) (closure progress_callback): data to pass * to @progress_callback * @cancellable: (nullable): a #GCancellable, or %NULL * @callback: function call when the asynchronous operation is complete * @user_data: data to pass to @callback * * Refresh any configured external appstream files, if the cache is too old. * * Since: 42 */ void gs_external_appstream_refresh_async (guint64 cache_age_secs, GsDownloadProgressCallback progress_callback, gpointer progress_user_data, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { g_autoptr(GSettings) settings = NULL; g_auto(GStrv) appstream_urls = NULL; gsize n_appstream_urls; g_autoptr(SoupSession) soup_session = NULL; g_autoptr(GTask) task = NULL; RefreshData *data; g_autoptr(RefreshData) data_owned = NULL; /* Chosen to allow a few UI updates per second without updating the * progress label so often it’s unreadable. */ const guint progress_update_period_ms = 300; task = g_task_new (NULL, cancellable, callback, user_data); g_task_set_source_tag (task, gs_external_appstream_refresh_async); settings = g_settings_new ("org.gnome.software"); soup_session = gs_build_soup_session (); appstream_urls = g_settings_get_strv (settings, "external-appstream-urls"); n_appstream_urls = g_strv_length (appstream_urls); data = data_owned = g_new0 (RefreshData, 1); data->progress_callback = progress_callback; data->progress_user_data = progress_user_data; data->n_appstream_urls = n_appstream_urls; data->progress_tuples = g_new0 (ProgressTuple, n_appstream_urls); data->progress_source = g_timeout_source_new (progress_update_period_ms); g_task_set_task_data (task, g_steal_pointer (&data_owned), (GDestroyNotify) refresh_data_free); /* Set up the progress timeout. This periodically sums up the progress * tuples in `data->progress_tuples` and reports them to the calling * function via @progress_callback, giving an overall progress for all * the parallel operations. */ g_source_set_callback (data->progress_source, progress_cb, g_object_ref (task), g_object_unref); g_source_attach (data->progress_source, g_main_context_get_thread_default ()); /* Refresh all the URIs in parallel. */ data->n_pending_ops = 1; for (gsize i = 0; i < n_appstream_urls; i++) { if (!g_str_has_prefix (appstream_urls[i], "https")) { g_warning ("Not considering %s as an external " "appstream source: please use an https URL", appstream_urls[i]); continue; } data->n_pending_ops++; refresh_url_async (settings, appstream_urls[i], soup_session, cache_age_secs, &data->progress_tuples[i], cancellable, refresh_cb, g_object_ref (task)); } finish_refresh_op (task, NULL); } static void refresh_cb (GObject *source_object, GAsyncResult *result, gpointer user_data) { g_autoptr(GTask) task = g_steal_pointer (&user_data); g_autoptr(GError) local_error = NULL; refresh_url_finish (result, &local_error); finish_refresh_op (task, g_steal_pointer (&local_error)); } static gboolean progress_cb (gpointer user_data) { GTask *task = G_TASK (user_data); RefreshData *data = g_task_get_task_data (task); gsize parallel_bytes_downloaded = 0, parallel_total_download_size = 0; /* Sum up the progress numerator and denominator for all parallel * downloads. */ for (gsize i = 0; i < data->n_appstream_urls; i++) { const ProgressTuple *progress_tuple = &data->progress_tuples[i]; if (!g_size_checked_add (¶llel_bytes_downloaded, parallel_bytes_downloaded, progress_tuple->bytes_downloaded)) parallel_bytes_downloaded = G_MAXSIZE; if (!g_size_checked_add (¶llel_total_download_size, parallel_total_download_size, progress_tuple->total_download_size)) parallel_total_download_size = G_MAXSIZE; } /* Report progress to the calling function. */ if (data->progress_callback != NULL) data->progress_callback (parallel_bytes_downloaded, parallel_total_download_size, data->progress_user_data); return G_SOURCE_CONTINUE; } /* @error is (transfer full) if non-%NULL */ static void finish_refresh_op (GTask *task, GError *error) { RefreshData *data = g_task_get_task_data (task); g_autoptr(GError) error_owned = g_steal_pointer (&error); if (data->error == NULL && error_owned != NULL) data->error = g_steal_pointer (&error_owned); else if (error_owned != NULL) g_debug ("Additional error while refreshing external appstream: %s", error_owned->message); g_assert (data->n_pending_ops > 0); data->n_pending_ops--; if (data->n_pending_ops > 0) return; /* Emit one final progress update, then stop any further ones. */ progress_cb (task); g_source_destroy (data->progress_source); /* All complete. */ if (data->error != NULL) g_task_return_error (task, g_steal_pointer (&data->error)); else g_task_return_boolean (task, TRUE); } /** * gs_external_appstream_refresh_finish: * @result: a #GAsyncResult * @error: return location for a #GError, or %NULL * * Finish an asynchronous refresh operation started with * gs_external_appstream_refresh_async(). * * Returns: %TRUE on success, %FALSE otherwise * Since: 42 */ gboolean gs_external_appstream_refresh_finish (GAsyncResult *result, GError **error) { g_return_val_if_fail (g_task_is_valid (result, NULL), FALSE); g_return_val_if_fail (error == NULL || *error == NULL, FALSE); return g_task_propagate_boolean (G_TASK (result), error); }