summaryrefslogtreecommitdiffstats
path: root/lib/gs-external-appstream-utils.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--lib/gs-external-appstream-utils.c627
1 files changed, 627 insertions, 0 deletions
diff --git a/lib/gs-external-appstream-utils.c b/lib/gs-external-appstream-utils.c
new file mode 100644
index 0000000..24ce3a8
--- /dev/null
+++ b/lib/gs-external-appstream-utils.c
@@ -0,0 +1,627 @@
+ /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2018 Endless Mobile, Inc.
+ *
+ * Authors: Joaquim Rocha <jrocha@endlessm.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+/*
+ * SECTION:gs-external-appstream-urls
+ * @short_description: Provides support for downloading external AppStream files.
+ *
+ * This downloads the set of configured external AppStream files, and caches
+ * them locally.
+ *
+ * According to the `external-appstream-system-wide` GSetting, the files will
+ * either be downloaded to a per-user cache, or to a system-wide cache. In the
+ * case of a system-wide cache, they are downloaded to a temporary file writable
+ * by the user, and then the suexec binary `gnome-software-install-appstream` is
+ * run to copy them to the system location.
+ *
+ * All the downloads are done in the default #GMainContext for the thread which
+ * calls gs_external_appstream_refresh_async(). They are done in parallel and
+ * the async refresh function will only complete once the last download is
+ * complete.
+ *
+ * Progress data is reported via a callback, and gives the total progress of all
+ * parallel downloads. Internally this is done by updating #ProgressTuple
+ * structs as each download progresses. A periodic timeout callback sums these
+ * and reports the total progress to the caller. That means that progress
+ * reports from gs_external_appstream_refresh_async() are done at a constant
+ * frequency.
+ *
+ * To test this code locally you will probably want to change your GSettings
+ * configuration to add some external AppStream URIs:
+ * ```
+ * gsettings set org.gnome.software external-appstream-urls '["https://example.com/appdata.xml.gz"]'
+ * ```
+ *
+ * When you are done with development, run the following command to use the real
+ * external AppStream list again:
+ * ```
+ * gsettings reset org.gnome.software external-appstream-urls
+ * ```
+ *
+ * Since: 42
+ */
+
+#include <errno.h>
+#include <glib.h>
+#include <glib/gi18n.h>
+#include <glib/gstdio.h>
+#include <libsoup/soup.h>
+
+#include "gs-external-appstream-utils.h"
+
+#define APPSTREAM_SYSTEM_DIR LOCALSTATEDIR "/cache/swcatalog/xml"
+
+G_DEFINE_QUARK (gs-external-appstream-error-quark, gs_external_appstream_error)
+
+gchar *
+gs_external_appstream_utils_get_file_cache_path (const gchar *file_name)
+{
+ g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s",
+ file_name);
+ return g_build_filename (APPSTREAM_SYSTEM_DIR, prefixed_file_name, NULL);
+}
+
+/* To be able to delete old files, when the path changed */
+gchar *
+gs_external_appstream_utils_get_legacy_file_cache_path (const gchar *file_name)
+{
+ g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s",
+ file_name);
+ return g_build_filename (LOCALSTATEDIR "/cache/app-info/xmls", prefixed_file_name, NULL);
+}
+
+const gchar *
+gs_external_appstream_utils_get_system_dir (void)
+{
+ return APPSTREAM_SYSTEM_DIR;
+}
+
+static gboolean
+gs_external_appstream_check (GFile *appstream_file,
+ guint64 cache_age_secs)
+{
+ guint64 appstream_file_age = gs_utils_get_file_age (appstream_file);
+ return appstream_file_age >= cache_age_secs;
+}
+
+static gboolean
+gs_external_appstream_install (const gchar *appstream_file,
+ GCancellable *cancellable,
+ GError **error)
+{
+ g_autoptr(GSubprocess) subprocess = NULL;
+ const gchar *argv[] = { "pkexec",
+ LIBEXECDIR "/gnome-software-install-appstream",
+ appstream_file, NULL};
+ g_debug ("Installing the appstream file %s in the system",
+ appstream_file);
+ subprocess = g_subprocess_newv (argv,
+ G_SUBPROCESS_FLAGS_STDOUT_PIPE |
+ G_SUBPROCESS_FLAGS_STDIN_PIPE, error);
+ if (subprocess == NULL)
+ return FALSE;
+ return g_subprocess_wait_check (subprocess, cancellable, error);
+}
+
+static void download_replace_file_cb (GObject *source_object,
+ GAsyncResult *result,
+ gpointer user_data);
+static void download_stream_cb (GObject *source_object,
+ GAsyncResult *result,
+ gpointer user_data);
+
+/* A tuple to store the last-received progress data for a single download.
+ * Each download (refresh_url_async()) has a pointer to the relevant
+ * #ProgressTuple for its download. These are stored in an array in #RefreshData
+ * and a timeout callback periodically sums them all and reports progress to the
+ * caller. */
+typedef struct {
+ gsize bytes_downloaded;
+ gsize total_download_size;
+} ProgressTuple;
+
+typedef struct {
+ /* Input data. */
+ gchar *url; /* (not nullable) (owned) */
+ GTask *task; /* (not nullable) (owned) */
+ GFile *output_file; /* (not nullable) (owned) */
+ ProgressTuple *progress_tuple; /* (not nullable) */
+ SoupSession *soup_session; /* (not nullable) (owned) */
+ gboolean system_wide;
+
+ /* In-progress data. */
+ gchar *last_etag; /* (nullable) (owned) */
+ GDateTime *last_modified_date; /* (nullable) (owned) */
+} DownloadAppStreamData;
+
+static void
+download_appstream_data_free (DownloadAppStreamData *data)
+{
+ g_free (data->url);
+ g_clear_object (&data->task);
+ g_clear_object (&data->output_file);
+ g_clear_object (&data->soup_session);
+ g_free (data->last_etag);
+ g_clear_pointer (&data->last_modified_date, g_date_time_unref);
+ g_free (data);
+}
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (DownloadAppStreamData, download_appstream_data_free)
+
+static void
+refresh_url_progress_cb (gsize bytes_downloaded,
+ gsize total_download_size,
+ gpointer user_data)
+{
+ ProgressTuple *tuple = user_data;
+
+ tuple->bytes_downloaded = bytes_downloaded;
+ tuple->total_download_size = total_download_size;
+
+ /* The timeout callback in progress_cb() periodically sums these. No
+ * need to notify of progress from here. */
+}
+
+static void
+refresh_url_async (GSettings *settings,
+ const gchar *url,
+ SoupSession *soup_session,
+ guint64 cache_age_secs,
+ ProgressTuple *progress_tuple,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
+{
+ g_autoptr(GTask) task = NULL;
+ g_autofree gchar *basename = NULL;
+ g_autofree gchar *basename_url = g_path_get_basename (url);
+ /* make sure different uris with same basenames differ */
+ g_autofree gchar *hash = NULL;
+ g_autofree gchar *target_file_path = NULL;
+ g_autoptr(GFile) target_file = NULL;
+ g_autoptr(GFile) tmp_file_parent = NULL;
+ g_autoptr(GFile) tmp_file = NULL;
+ g_autoptr(GsApp) app_dl = gs_app_new ("external-appstream");
+ g_autoptr(GError) local_error = NULL;
+ DownloadAppStreamData *data;
+ gboolean system_wide;
+
+ task = g_task_new (NULL, cancellable, callback, user_data);
+ g_task_set_source_tag (task, refresh_url_async);
+
+ /* Calculate the basename of the target file. */
+ hash = g_compute_checksum_for_string (G_CHECKSUM_SHA1, url, -1);
+ if (hash == NULL) {
+ g_task_return_new_error (task,
+ GS_EXTERNAL_APPSTREAM_ERROR,
+ GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING,
+ "Failed to hash URI ‘%s’", url);
+ return;
+ }
+ basename = g_strdup_printf ("%s-%s", hash, basename_url);
+
+ /* Are we downloading for the user, or the system? */
+ system_wide = g_settings_get_boolean (settings, "external-appstream-system-wide");
+
+ /* Check cache file age. */
+ if (system_wide) {
+ target_file_path = gs_external_appstream_utils_get_file_cache_path (basename);
+ } else {
+ g_autofree gchar *legacy_file_path = NULL;
+
+ target_file_path = g_build_filename (g_get_user_data_dir (),
+ "swcatalog",
+ "xml",
+ basename,
+ NULL);
+
+ /* Delete an old file, from a legacy location */
+ legacy_file_path = g_build_filename (g_get_user_data_dir (),
+ "app-info",
+ "xmls",
+ basename,
+ NULL);
+
+ if (g_unlink (legacy_file_path) == -1) {
+ int errn = errno;
+ if (errn != ENOENT)
+ g_debug ("Failed to unlink '%s': %s", legacy_file_path, g_strerror (errn));
+
+ }
+ }
+
+ target_file = g_file_new_for_path (target_file_path);
+
+ if (!gs_external_appstream_check (target_file, cache_age_secs)) {
+ g_debug ("skipping updating external appstream file %s: "
+ "cache age is older than file",
+ target_file_path);
+ g_task_return_boolean (task, TRUE);
+ return;
+ }
+
+ /* If downloading system wide, write the download contents into a
+ * temporary file that will be copied into the system location later. */
+ if (system_wide) {
+ g_autofree gchar *tmp_file_path = NULL;
+
+ tmp_file_path = gs_utils_get_cache_filename ("external-appstream",
+ basename,
+ GS_UTILS_CACHE_FLAG_WRITEABLE |
+ GS_UTILS_CACHE_FLAG_CREATE_DIRECTORY,
+ &local_error);
+ if (tmp_file_path == NULL) {
+ g_task_return_error (task, g_steal_pointer (&local_error));
+ return;
+ }
+
+ tmp_file = g_file_new_for_path (tmp_file_path);
+ } else {
+ tmp_file = g_object_ref (target_file);
+ }
+
+ gs_app_set_summary_missing (app_dl,
+ /* TRANSLATORS: status text when downloading */
+ _("Downloading extra metadata files…"));
+
+ data = g_new0 (DownloadAppStreamData, 1);
+ data->url = g_strdup (url);
+ data->task = g_object_ref (task);
+ data->output_file = g_object_ref (tmp_file);
+ data->progress_tuple = progress_tuple;
+ data->soup_session = g_object_ref (soup_session);
+ data->system_wide = system_wide;
+ g_task_set_task_data (task, data, (GDestroyNotify) download_appstream_data_free);
+
+ /* Create the destination file’s directory.
+ * FIXME: This should be made async; it hasn’t done for now as it’s
+ * likely to be fast. */
+ tmp_file_parent = g_file_get_parent (tmp_file);
+
+ if (tmp_file_parent != NULL &&
+ !g_file_make_directory_with_parents (tmp_file_parent, cancellable, &local_error) &&
+ !g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_EXISTS)) {
+ g_task_return_error (task, g_steal_pointer (&local_error));
+ return;
+ }
+
+ g_clear_error (&local_error);
+
+ /* Query the ETag and modification date of the target file, if the file already exists. For
+ * system-wide installations, this is the ETag of the AppStream file installed system-wide.
+ * For local installations, this is just the local output file. */
+ data->last_etag = gs_utils_get_file_etag (target_file, &data->last_modified_date, cancellable);
+ g_debug ("Queried ETag of file %s: %s", g_file_peek_path (target_file), data->last_etag);
+
+ /* Create the output file */
+ g_file_replace_async (tmp_file,
+ NULL, /* ETag */
+ FALSE, /* make_backup */
+ G_FILE_CREATE_PRIVATE | G_FILE_CREATE_REPLACE_DESTINATION,
+ G_PRIORITY_LOW,
+ cancellable,
+ download_replace_file_cb,
+ g_steal_pointer (&task));
+}
+
+static void
+download_replace_file_cb (GObject *source_object,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ GFile *output_file = G_FILE (source_object);
+ g_autoptr(GTask) task = g_steal_pointer (&user_data);
+ GCancellable *cancellable = g_task_get_cancellable (task);
+ DownloadAppStreamData *data = g_task_get_task_data (task);
+ g_autoptr(GFileOutputStream) output_stream = NULL;
+ g_autoptr(GError) local_error = NULL;
+
+ output_stream = g_file_replace_finish (output_file, result, &local_error);
+
+ if (output_stream == NULL) {
+ g_task_return_error (task, g_steal_pointer (&local_error));
+ return;
+ }
+
+ /* Do the download. */
+ gs_download_stream_async (data->soup_session,
+ data->url,
+ G_OUTPUT_STREAM (output_stream),
+ data->last_etag,
+ data->last_modified_date,
+ G_PRIORITY_LOW,
+ refresh_url_progress_cb,
+ data->progress_tuple,
+ cancellable,
+ download_stream_cb,
+ g_steal_pointer (&task));
+}
+
+static void
+download_stream_cb (GObject *source_object,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ SoupSession *soup_session = SOUP_SESSION (source_object);
+ g_autoptr(GTask) task = g_steal_pointer (&user_data);
+ GCancellable *cancellable = g_task_get_cancellable (task);
+ DownloadAppStreamData *data = g_task_get_task_data (task);
+ g_autoptr(GError) local_error = NULL;
+ g_autofree gchar *new_etag = NULL;
+
+ if (!gs_download_stream_finish (soup_session, result, &new_etag, NULL, &local_error)) {
+ if (data->system_wide && g_error_matches (local_error, GS_DOWNLOAD_ERROR, GS_DOWNLOAD_ERROR_NOT_MODIFIED)) {
+ g_debug ("External AppStream file not modified, removing temporary download file %s",
+ g_file_peek_path (data->output_file));
+
+ /* System-wide installs should delete the empty file created when preparing to
+ * download the external AppStream file. */
+ g_file_delete_async (data->output_file, G_PRIORITY_LOW, NULL, NULL, NULL);
+ g_task_return_boolean (task, TRUE);
+ } else if (!g_network_monitor_get_network_available (g_network_monitor_get_default ())) {
+ g_task_return_new_error (task,
+ GS_EXTERNAL_APPSTREAM_ERROR,
+ GS_EXTERNAL_APPSTREAM_ERROR_NO_NETWORK,
+ "External AppStream could not be downloaded due to being offline");
+ } else {
+ g_task_return_new_error (task,
+ GS_EXTERNAL_APPSTREAM_ERROR,
+ GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING,
+ "Server returned no data for external AppStream file: %s",
+ local_error->message);
+ }
+ return;
+ }
+
+ g_debug ("Downloaded appstream file %s", g_file_peek_path (data->output_file));
+
+ gs_utils_set_file_etag (data->output_file, new_etag, cancellable);
+
+ if (data->system_wide) {
+ /* install file systemwide */
+ if (!gs_external_appstream_install (g_file_peek_path (data->output_file),
+ cancellable,
+ &local_error)) {
+ g_task_return_new_error (task,
+ GS_EXTERNAL_APPSTREAM_ERROR,
+ GS_EXTERNAL_APPSTREAM_ERROR_INSTALLING_ON_SYSTEM,
+ "Error installing external AppStream file on system: %s", local_error->message);
+ return;
+ }
+ g_debug ("Installed appstream file %s", g_file_peek_path (data->output_file));
+ }
+
+ g_task_return_boolean (task, TRUE);
+}
+
+static gboolean
+refresh_url_finish (GAsyncResult *result,
+ GError **error)
+{
+ return g_task_propagate_boolean (G_TASK (result), error);
+}
+
+static void refresh_cb (GObject *source_object,
+ GAsyncResult *result,
+ gpointer user_data);
+static gboolean progress_cb (gpointer user_data);
+static void finish_refresh_op (GTask *task,
+ GError *error);
+
+typedef struct {
+ /* Input data. */
+ guint64 cache_age_secs;
+
+ /* In-progress data. */
+ guint n_pending_ops;
+ GError *error; /* (nullable) (owned) */
+ gsize n_appstream_urls;
+ GsDownloadProgressCallback progress_callback; /* (nullable) */
+ gpointer progress_user_data; /* (closure progress_callback) */
+ ProgressTuple *progress_tuples; /* (array length=n_appstream_urls) (owned) */
+ GSource *progress_source; /* (owned) */
+} RefreshData;
+
+static void
+refresh_data_free (RefreshData *data)
+{
+ g_assert (data->n_pending_ops == 0);
+
+ /* If this was set it should have been stolen for g_task_return_error()
+ * by now. */
+ g_assert (data->error == NULL);
+
+ /* Similarly, progress reporting should have been stopped by now. */
+ g_assert (g_source_is_destroyed (data->progress_source));
+ g_source_unref (data->progress_source);
+
+ g_free (data->progress_tuples);
+
+ g_free (data);
+}
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (RefreshData, refresh_data_free)
+
+/**
+ * gs_external_appstream_refresh_async:
+ * @cache_age_secs: cache age, in seconds, as passed to #GsPluginClass.refresh_metadata_async()
+ * @progress_callback: (nullable): callback to call with progress information
+ * @progress_user_data: (nullable) (closure progress_callback): data to pass
+ * to @progress_callback
+ * @cancellable: (nullable): a #GCancellable, or %NULL
+ * @callback: function call when the asynchronous operation is complete
+ * @user_data: data to pass to @callback
+ *
+ * Refresh any configured external appstream files, if the cache is too old.
+ *
+ * Since: 42
+ */
+void
+gs_external_appstream_refresh_async (guint64 cache_age_secs,
+ GsDownloadProgressCallback progress_callback,
+ gpointer progress_user_data,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
+{
+ g_autoptr(GSettings) settings = NULL;
+ g_auto(GStrv) appstream_urls = NULL;
+ gsize n_appstream_urls;
+ g_autoptr(SoupSession) soup_session = NULL;
+ g_autoptr(GTask) task = NULL;
+ RefreshData *data;
+ g_autoptr(RefreshData) data_owned = NULL;
+
+ /* Chosen to allow a few UI updates per second without updating the
+ * progress label so often it’s unreadable. */
+ const guint progress_update_period_ms = 300;
+
+ task = g_task_new (NULL, cancellable, callback, user_data);
+ g_task_set_source_tag (task, gs_external_appstream_refresh_async);
+
+ settings = g_settings_new ("org.gnome.software");
+ soup_session = gs_build_soup_session ();
+ appstream_urls = g_settings_get_strv (settings,
+ "external-appstream-urls");
+ n_appstream_urls = g_strv_length (appstream_urls);
+
+ data = data_owned = g_new0 (RefreshData, 1);
+ data->progress_callback = progress_callback;
+ data->progress_user_data = progress_user_data;
+ data->n_appstream_urls = n_appstream_urls;
+ data->progress_tuples = g_new0 (ProgressTuple, n_appstream_urls);
+ data->progress_source = g_timeout_source_new (progress_update_period_ms);
+ g_task_set_task_data (task, g_steal_pointer (&data_owned), (GDestroyNotify) refresh_data_free);
+
+ /* Set up the progress timeout. This periodically sums up the progress
+ * tuples in `data->progress_tuples` and reports them to the calling
+ * function via @progress_callback, giving an overall progress for all
+ * the parallel operations. */
+ g_source_set_callback (data->progress_source, progress_cb, g_object_ref (task), g_object_unref);
+ g_source_attach (data->progress_source, g_main_context_get_thread_default ());
+
+ /* Refresh all the URIs in parallel. */
+ data->n_pending_ops = 1;
+
+ for (gsize i = 0; i < n_appstream_urls; i++) {
+ if (!g_str_has_prefix (appstream_urls[i], "https")) {
+ g_warning ("Not considering %s as an external "
+ "appstream source: please use an https URL",
+ appstream_urls[i]);
+ continue;
+ }
+
+ data->n_pending_ops++;
+ refresh_url_async (settings,
+ appstream_urls[i],
+ soup_session,
+ cache_age_secs,
+ &data->progress_tuples[i],
+ cancellable,
+ refresh_cb,
+ g_object_ref (task));
+ }
+
+ finish_refresh_op (task, NULL);
+}
+
+static void
+refresh_cb (GObject *source_object,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ g_autoptr(GTask) task = g_steal_pointer (&user_data);
+ g_autoptr(GError) local_error = NULL;
+
+ refresh_url_finish (result, &local_error);
+ finish_refresh_op (task, g_steal_pointer (&local_error));
+}
+
+static gboolean
+progress_cb (gpointer user_data)
+{
+ GTask *task = G_TASK (user_data);
+ RefreshData *data = g_task_get_task_data (task);
+ gsize parallel_bytes_downloaded = 0, parallel_total_download_size = 0;
+
+ /* Sum up the progress numerator and denominator for all parallel
+ * downloads. */
+ for (gsize i = 0; i < data->n_appstream_urls; i++) {
+ const ProgressTuple *progress_tuple = &data->progress_tuples[i];
+
+ if (!g_size_checked_add (&parallel_bytes_downloaded,
+ parallel_bytes_downloaded,
+ progress_tuple->bytes_downloaded))
+ parallel_bytes_downloaded = G_MAXSIZE;
+ if (!g_size_checked_add (&parallel_total_download_size,
+ parallel_total_download_size,
+ progress_tuple->total_download_size))
+ parallel_total_download_size = G_MAXSIZE;
+ }
+
+ /* Report progress to the calling function. */
+ if (data->progress_callback != NULL)
+ data->progress_callback (parallel_bytes_downloaded,
+ parallel_total_download_size,
+ data->progress_user_data);
+
+ return G_SOURCE_CONTINUE;
+}
+
+/* @error is (transfer full) if non-%NULL */
+static void
+finish_refresh_op (GTask *task,
+ GError *error)
+{
+ RefreshData *data = g_task_get_task_data (task);
+ g_autoptr(GError) error_owned = g_steal_pointer (&error);
+
+ if (data->error == NULL && error_owned != NULL)
+ data->error = g_steal_pointer (&error_owned);
+ else if (error_owned != NULL)
+ g_debug ("Additional error while refreshing external appstream: %s", error_owned->message);
+
+ g_assert (data->n_pending_ops > 0);
+ data->n_pending_ops--;
+
+ if (data->n_pending_ops > 0)
+ return;
+
+ /* Emit one final progress update, then stop any further ones. */
+ progress_cb (task);
+ g_source_destroy (data->progress_source);
+
+ /* All complete. */
+ if (data->error != NULL)
+ g_task_return_error (task, g_steal_pointer (&data->error));
+ else
+ g_task_return_boolean (task, TRUE);
+}
+
+/**
+ * gs_external_appstream_refresh_finish:
+ * @result: a #GAsyncResult
+ * @error: return location for a #GError, or %NULL
+ *
+ * Finish an asynchronous refresh operation started with
+ * gs_external_appstream_refresh_async().
+ *
+ * Returns: %TRUE on success, %FALSE otherwise
+ * Since: 42
+ */
+gboolean
+gs_external_appstream_refresh_finish (GAsyncResult *result,
+ GError **error)
+{
+ g_return_val_if_fail (g_task_is_valid (result, NULL), FALSE);
+ g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
+
+ return g_task_propagate_boolean (G_TASK (result), error);
+}