diff options
Diffstat (limited to 'lib/gs-external-appstream-utils.c')
| -rw-r--r-- | lib/gs-external-appstream-utils.c | 627 |
1 files changed, 627 insertions, 0 deletions
diff --git a/lib/gs-external-appstream-utils.c b/lib/gs-external-appstream-utils.c new file mode 100644 index 0000000..24ce3a8 --- /dev/null +++ b/lib/gs-external-appstream-utils.c @@ -0,0 +1,627 @@ + /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- + * vi:set noexpandtab tabstop=8 shiftwidth=8: + * + * Copyright (C) 2018 Endless Mobile, Inc. + * + * Authors: Joaquim Rocha <jrocha@endlessm.com> + * + * SPDX-License-Identifier: GPL-2.0+ + */ + +/* + * SECTION:gs-external-appstream-urls + * @short_description: Provides support for downloading external AppStream files. + * + * This downloads the set of configured external AppStream files, and caches + * them locally. + * + * According to the `external-appstream-system-wide` GSetting, the files will + * either be downloaded to a per-user cache, or to a system-wide cache. In the + * case of a system-wide cache, they are downloaded to a temporary file writable + * by the user, and then the suexec binary `gnome-software-install-appstream` is + * run to copy them to the system location. + * + * All the downloads are done in the default #GMainContext for the thread which + * calls gs_external_appstream_refresh_async(). They are done in parallel and + * the async refresh function will only complete once the last download is + * complete. + * + * Progress data is reported via a callback, and gives the total progress of all + * parallel downloads. Internally this is done by updating #ProgressTuple + * structs as each download progresses. A periodic timeout callback sums these + * and reports the total progress to the caller. That means that progress + * reports from gs_external_appstream_refresh_async() are done at a constant + * frequency. + * + * To test this code locally you will probably want to change your GSettings + * configuration to add some external AppStream URIs: + * ``` + * gsettings set org.gnome.software external-appstream-urls '["https://example.com/appdata.xml.gz"]' + * ``` + * + * When you are done with development, run the following command to use the real + * external AppStream list again: + * ``` + * gsettings reset org.gnome.software external-appstream-urls + * ``` + * + * Since: 42 + */ + +#include <errno.h> +#include <glib.h> +#include <glib/gi18n.h> +#include <glib/gstdio.h> +#include <libsoup/soup.h> + +#include "gs-external-appstream-utils.h" + +#define APPSTREAM_SYSTEM_DIR LOCALSTATEDIR "/cache/swcatalog/xml" + +G_DEFINE_QUARK (gs-external-appstream-error-quark, gs_external_appstream_error) + +gchar * +gs_external_appstream_utils_get_file_cache_path (const gchar *file_name) +{ + g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s", + file_name); + return g_build_filename (APPSTREAM_SYSTEM_DIR, prefixed_file_name, NULL); +} + +/* To be able to delete old files, when the path changed */ +gchar * +gs_external_appstream_utils_get_legacy_file_cache_path (const gchar *file_name) +{ + g_autofree gchar *prefixed_file_name = g_strdup_printf (EXTERNAL_APPSTREAM_PREFIX "-%s", + file_name); + return g_build_filename (LOCALSTATEDIR "/cache/app-info/xmls", prefixed_file_name, NULL); +} + +const gchar * +gs_external_appstream_utils_get_system_dir (void) +{ + return APPSTREAM_SYSTEM_DIR; +} + +static gboolean +gs_external_appstream_check (GFile *appstream_file, + guint64 cache_age_secs) +{ + guint64 appstream_file_age = gs_utils_get_file_age (appstream_file); + return appstream_file_age >= cache_age_secs; +} + +static gboolean +gs_external_appstream_install (const gchar *appstream_file, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GSubprocess) subprocess = NULL; + const gchar *argv[] = { "pkexec", + LIBEXECDIR "/gnome-software-install-appstream", + appstream_file, NULL}; + g_debug ("Installing the appstream file %s in the system", + appstream_file); + subprocess = g_subprocess_newv (argv, + G_SUBPROCESS_FLAGS_STDOUT_PIPE | + G_SUBPROCESS_FLAGS_STDIN_PIPE, error); + if (subprocess == NULL) + return FALSE; + return g_subprocess_wait_check (subprocess, cancellable, error); +} + +static void download_replace_file_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data); +static void download_stream_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data); + +/* A tuple to store the last-received progress data for a single download. + * Each download (refresh_url_async()) has a pointer to the relevant + * #ProgressTuple for its download. These are stored in an array in #RefreshData + * and a timeout callback periodically sums them all and reports progress to the + * caller. */ +typedef struct { + gsize bytes_downloaded; + gsize total_download_size; +} ProgressTuple; + +typedef struct { + /* Input data. */ + gchar *url; /* (not nullable) (owned) */ + GTask *task; /* (not nullable) (owned) */ + GFile *output_file; /* (not nullable) (owned) */ + ProgressTuple *progress_tuple; /* (not nullable) */ + SoupSession *soup_session; /* (not nullable) (owned) */ + gboolean system_wide; + + /* In-progress data. */ + gchar *last_etag; /* (nullable) (owned) */ + GDateTime *last_modified_date; /* (nullable) (owned) */ +} DownloadAppStreamData; + +static void +download_appstream_data_free (DownloadAppStreamData *data) +{ + g_free (data->url); + g_clear_object (&data->task); + g_clear_object (&data->output_file); + g_clear_object (&data->soup_session); + g_free (data->last_etag); + g_clear_pointer (&data->last_modified_date, g_date_time_unref); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (DownloadAppStreamData, download_appstream_data_free) + +static void +refresh_url_progress_cb (gsize bytes_downloaded, + gsize total_download_size, + gpointer user_data) +{ + ProgressTuple *tuple = user_data; + + tuple->bytes_downloaded = bytes_downloaded; + tuple->total_download_size = total_download_size; + + /* The timeout callback in progress_cb() periodically sums these. No + * need to notify of progress from here. */ +} + +static void +refresh_url_async (GSettings *settings, + const gchar *url, + SoupSession *soup_session, + guint64 cache_age_secs, + ProgressTuple *progress_tuple, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autofree gchar *basename = NULL; + g_autofree gchar *basename_url = g_path_get_basename (url); + /* make sure different uris with same basenames differ */ + g_autofree gchar *hash = NULL; + g_autofree gchar *target_file_path = NULL; + g_autoptr(GFile) target_file = NULL; + g_autoptr(GFile) tmp_file_parent = NULL; + g_autoptr(GFile) tmp_file = NULL; + g_autoptr(GsApp) app_dl = gs_app_new ("external-appstream"); + g_autoptr(GError) local_error = NULL; + DownloadAppStreamData *data; + gboolean system_wide; + + task = g_task_new (NULL, cancellable, callback, user_data); + g_task_set_source_tag (task, refresh_url_async); + + /* Calculate the basename of the target file. */ + hash = g_compute_checksum_for_string (G_CHECKSUM_SHA1, url, -1); + if (hash == NULL) { + g_task_return_new_error (task, + GS_EXTERNAL_APPSTREAM_ERROR, + GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING, + "Failed to hash URI ‘%s’", url); + return; + } + basename = g_strdup_printf ("%s-%s", hash, basename_url); + + /* Are we downloading for the user, or the system? */ + system_wide = g_settings_get_boolean (settings, "external-appstream-system-wide"); + + /* Check cache file age. */ + if (system_wide) { + target_file_path = gs_external_appstream_utils_get_file_cache_path (basename); + } else { + g_autofree gchar *legacy_file_path = NULL; + + target_file_path = g_build_filename (g_get_user_data_dir (), + "swcatalog", + "xml", + basename, + NULL); + + /* Delete an old file, from a legacy location */ + legacy_file_path = g_build_filename (g_get_user_data_dir (), + "app-info", + "xmls", + basename, + NULL); + + if (g_unlink (legacy_file_path) == -1) { + int errn = errno; + if (errn != ENOENT) + g_debug ("Failed to unlink '%s': %s", legacy_file_path, g_strerror (errn)); + + } + } + + target_file = g_file_new_for_path (target_file_path); + + if (!gs_external_appstream_check (target_file, cache_age_secs)) { + g_debug ("skipping updating external appstream file %s: " + "cache age is older than file", + target_file_path); + g_task_return_boolean (task, TRUE); + return; + } + + /* If downloading system wide, write the download contents into a + * temporary file that will be copied into the system location later. */ + if (system_wide) { + g_autofree gchar *tmp_file_path = NULL; + + tmp_file_path = gs_utils_get_cache_filename ("external-appstream", + basename, + GS_UTILS_CACHE_FLAG_WRITEABLE | + GS_UTILS_CACHE_FLAG_CREATE_DIRECTORY, + &local_error); + if (tmp_file_path == NULL) { + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + tmp_file = g_file_new_for_path (tmp_file_path); + } else { + tmp_file = g_object_ref (target_file); + } + + gs_app_set_summary_missing (app_dl, + /* TRANSLATORS: status text when downloading */ + _("Downloading extra metadata files…")); + + data = g_new0 (DownloadAppStreamData, 1); + data->url = g_strdup (url); + data->task = g_object_ref (task); + data->output_file = g_object_ref (tmp_file); + data->progress_tuple = progress_tuple; + data->soup_session = g_object_ref (soup_session); + data->system_wide = system_wide; + g_task_set_task_data (task, data, (GDestroyNotify) download_appstream_data_free); + + /* Create the destination file’s directory. + * FIXME: This should be made async; it hasn’t done for now as it’s + * likely to be fast. */ + tmp_file_parent = g_file_get_parent (tmp_file); + + if (tmp_file_parent != NULL && + !g_file_make_directory_with_parents (tmp_file_parent, cancellable, &local_error) && + !g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_EXISTS)) { + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + g_clear_error (&local_error); + + /* Query the ETag and modification date of the target file, if the file already exists. For + * system-wide installations, this is the ETag of the AppStream file installed system-wide. + * For local installations, this is just the local output file. */ + data->last_etag = gs_utils_get_file_etag (target_file, &data->last_modified_date, cancellable); + g_debug ("Queried ETag of file %s: %s", g_file_peek_path (target_file), data->last_etag); + + /* Create the output file */ + g_file_replace_async (tmp_file, + NULL, /* ETag */ + FALSE, /* make_backup */ + G_FILE_CREATE_PRIVATE | G_FILE_CREATE_REPLACE_DESTINATION, + G_PRIORITY_LOW, + cancellable, + download_replace_file_cb, + g_steal_pointer (&task)); +} + +static void +download_replace_file_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data) +{ + GFile *output_file = G_FILE (source_object); + g_autoptr(GTask) task = g_steal_pointer (&user_data); + GCancellable *cancellable = g_task_get_cancellable (task); + DownloadAppStreamData *data = g_task_get_task_data (task); + g_autoptr(GFileOutputStream) output_stream = NULL; + g_autoptr(GError) local_error = NULL; + + output_stream = g_file_replace_finish (output_file, result, &local_error); + + if (output_stream == NULL) { + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + /* Do the download. */ + gs_download_stream_async (data->soup_session, + data->url, + G_OUTPUT_STREAM (output_stream), + data->last_etag, + data->last_modified_date, + G_PRIORITY_LOW, + refresh_url_progress_cb, + data->progress_tuple, + cancellable, + download_stream_cb, + g_steal_pointer (&task)); +} + +static void +download_stream_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data) +{ + SoupSession *soup_session = SOUP_SESSION (source_object); + g_autoptr(GTask) task = g_steal_pointer (&user_data); + GCancellable *cancellable = g_task_get_cancellable (task); + DownloadAppStreamData *data = g_task_get_task_data (task); + g_autoptr(GError) local_error = NULL; + g_autofree gchar *new_etag = NULL; + + if (!gs_download_stream_finish (soup_session, result, &new_etag, NULL, &local_error)) { + if (data->system_wide && g_error_matches (local_error, GS_DOWNLOAD_ERROR, GS_DOWNLOAD_ERROR_NOT_MODIFIED)) { + g_debug ("External AppStream file not modified, removing temporary download file %s", + g_file_peek_path (data->output_file)); + + /* System-wide installs should delete the empty file created when preparing to + * download the external AppStream file. */ + g_file_delete_async (data->output_file, G_PRIORITY_LOW, NULL, NULL, NULL); + g_task_return_boolean (task, TRUE); + } else if (!g_network_monitor_get_network_available (g_network_monitor_get_default ())) { + g_task_return_new_error (task, + GS_EXTERNAL_APPSTREAM_ERROR, + GS_EXTERNAL_APPSTREAM_ERROR_NO_NETWORK, + "External AppStream could not be downloaded due to being offline"); + } else { + g_task_return_new_error (task, + GS_EXTERNAL_APPSTREAM_ERROR, + GS_EXTERNAL_APPSTREAM_ERROR_DOWNLOADING, + "Server returned no data for external AppStream file: %s", + local_error->message); + } + return; + } + + g_debug ("Downloaded appstream file %s", g_file_peek_path (data->output_file)); + + gs_utils_set_file_etag (data->output_file, new_etag, cancellable); + + if (data->system_wide) { + /* install file systemwide */ + if (!gs_external_appstream_install (g_file_peek_path (data->output_file), + cancellable, + &local_error)) { + g_task_return_new_error (task, + GS_EXTERNAL_APPSTREAM_ERROR, + GS_EXTERNAL_APPSTREAM_ERROR_INSTALLING_ON_SYSTEM, + "Error installing external AppStream file on system: %s", local_error->message); + return; + } + g_debug ("Installed appstream file %s", g_file_peek_path (data->output_file)); + } + + g_task_return_boolean (task, TRUE); +} + +static gboolean +refresh_url_finish (GAsyncResult *result, + GError **error) +{ + return g_task_propagate_boolean (G_TASK (result), error); +} + +static void refresh_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data); +static gboolean progress_cb (gpointer user_data); +static void finish_refresh_op (GTask *task, + GError *error); + +typedef struct { + /* Input data. */ + guint64 cache_age_secs; + + /* In-progress data. */ + guint n_pending_ops; + GError *error; /* (nullable) (owned) */ + gsize n_appstream_urls; + GsDownloadProgressCallback progress_callback; /* (nullable) */ + gpointer progress_user_data; /* (closure progress_callback) */ + ProgressTuple *progress_tuples; /* (array length=n_appstream_urls) (owned) */ + GSource *progress_source; /* (owned) */ +} RefreshData; + +static void +refresh_data_free (RefreshData *data) +{ + g_assert (data->n_pending_ops == 0); + + /* If this was set it should have been stolen for g_task_return_error() + * by now. */ + g_assert (data->error == NULL); + + /* Similarly, progress reporting should have been stopped by now. */ + g_assert (g_source_is_destroyed (data->progress_source)); + g_source_unref (data->progress_source); + + g_free (data->progress_tuples); + + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (RefreshData, refresh_data_free) + +/** + * gs_external_appstream_refresh_async: + * @cache_age_secs: cache age, in seconds, as passed to #GsPluginClass.refresh_metadata_async() + * @progress_callback: (nullable): callback to call with progress information + * @progress_user_data: (nullable) (closure progress_callback): data to pass + * to @progress_callback + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: function call when the asynchronous operation is complete + * @user_data: data to pass to @callback + * + * Refresh any configured external appstream files, if the cache is too old. + * + * Since: 42 + */ +void +gs_external_appstream_refresh_async (guint64 cache_age_secs, + GsDownloadProgressCallback progress_callback, + gpointer progress_user_data, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GSettings) settings = NULL; + g_auto(GStrv) appstream_urls = NULL; + gsize n_appstream_urls; + g_autoptr(SoupSession) soup_session = NULL; + g_autoptr(GTask) task = NULL; + RefreshData *data; + g_autoptr(RefreshData) data_owned = NULL; + + /* Chosen to allow a few UI updates per second without updating the + * progress label so often it’s unreadable. */ + const guint progress_update_period_ms = 300; + + task = g_task_new (NULL, cancellable, callback, user_data); + g_task_set_source_tag (task, gs_external_appstream_refresh_async); + + settings = g_settings_new ("org.gnome.software"); + soup_session = gs_build_soup_session (); + appstream_urls = g_settings_get_strv (settings, + "external-appstream-urls"); + n_appstream_urls = g_strv_length (appstream_urls); + + data = data_owned = g_new0 (RefreshData, 1); + data->progress_callback = progress_callback; + data->progress_user_data = progress_user_data; + data->n_appstream_urls = n_appstream_urls; + data->progress_tuples = g_new0 (ProgressTuple, n_appstream_urls); + data->progress_source = g_timeout_source_new (progress_update_period_ms); + g_task_set_task_data (task, g_steal_pointer (&data_owned), (GDestroyNotify) refresh_data_free); + + /* Set up the progress timeout. This periodically sums up the progress + * tuples in `data->progress_tuples` and reports them to the calling + * function via @progress_callback, giving an overall progress for all + * the parallel operations. */ + g_source_set_callback (data->progress_source, progress_cb, g_object_ref (task), g_object_unref); + g_source_attach (data->progress_source, g_main_context_get_thread_default ()); + + /* Refresh all the URIs in parallel. */ + data->n_pending_ops = 1; + + for (gsize i = 0; i < n_appstream_urls; i++) { + if (!g_str_has_prefix (appstream_urls[i], "https")) { + g_warning ("Not considering %s as an external " + "appstream source: please use an https URL", + appstream_urls[i]); + continue; + } + + data->n_pending_ops++; + refresh_url_async (settings, + appstream_urls[i], + soup_session, + cache_age_secs, + &data->progress_tuples[i], + cancellable, + refresh_cb, + g_object_ref (task)); + } + + finish_refresh_op (task, NULL); +} + +static void +refresh_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data) +{ + g_autoptr(GTask) task = g_steal_pointer (&user_data); + g_autoptr(GError) local_error = NULL; + + refresh_url_finish (result, &local_error); + finish_refresh_op (task, g_steal_pointer (&local_error)); +} + +static gboolean +progress_cb (gpointer user_data) +{ + GTask *task = G_TASK (user_data); + RefreshData *data = g_task_get_task_data (task); + gsize parallel_bytes_downloaded = 0, parallel_total_download_size = 0; + + /* Sum up the progress numerator and denominator for all parallel + * downloads. */ + for (gsize i = 0; i < data->n_appstream_urls; i++) { + const ProgressTuple *progress_tuple = &data->progress_tuples[i]; + + if (!g_size_checked_add (¶llel_bytes_downloaded, + parallel_bytes_downloaded, + progress_tuple->bytes_downloaded)) + parallel_bytes_downloaded = G_MAXSIZE; + if (!g_size_checked_add (¶llel_total_download_size, + parallel_total_download_size, + progress_tuple->total_download_size)) + parallel_total_download_size = G_MAXSIZE; + } + + /* Report progress to the calling function. */ + if (data->progress_callback != NULL) + data->progress_callback (parallel_bytes_downloaded, + parallel_total_download_size, + data->progress_user_data); + + return G_SOURCE_CONTINUE; +} + +/* @error is (transfer full) if non-%NULL */ +static void +finish_refresh_op (GTask *task, + GError *error) +{ + RefreshData *data = g_task_get_task_data (task); + g_autoptr(GError) error_owned = g_steal_pointer (&error); + + if (data->error == NULL && error_owned != NULL) + data->error = g_steal_pointer (&error_owned); + else if (error_owned != NULL) + g_debug ("Additional error while refreshing external appstream: %s", error_owned->message); + + g_assert (data->n_pending_ops > 0); + data->n_pending_ops--; + + if (data->n_pending_ops > 0) + return; + + /* Emit one final progress update, then stop any further ones. */ + progress_cb (task); + g_source_destroy (data->progress_source); + + /* All complete. */ + if (data->error != NULL) + g_task_return_error (task, g_steal_pointer (&data->error)); + else + g_task_return_boolean (task, TRUE); +} + +/** + * gs_external_appstream_refresh_finish: + * @result: a #GAsyncResult + * @error: return location for a #GError, or %NULL + * + * Finish an asynchronous refresh operation started with + * gs_external_appstream_refresh_async(). + * + * Returns: %TRUE on success, %FALSE otherwise + * Since: 42 + */ +gboolean +gs_external_appstream_refresh_finish (GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, NULL), FALSE); + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + + return g_task_propagate_boolean (G_TASK (result), error); +} |