Adding upstream version 3.38.1.upstream/3.38.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

20 files changed, 5218 insertions, 0 deletions

diff --git a/plugins/core/gs-appstream.c b/plugins/core/gs-appstream.c
new file mode 100644
index 0000000..d514cbf
--- /dev/null
+++ b/plugins/core/gs-appstream.c
@@ -0,0 +1,1581 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2015-2017 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2018-2019 Kalev Lember <klember@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include "config.h"
+
+#include <gnome-software.h>
+
+#include "gs-appstream.h"
+
+#define	GS_APPSTREAM_MAX_SCREENSHOTS	5
+
+GsApp *
+gs_appstream_create_app (GsPlugin *plugin, XbSilo *silo, XbNode *component, GError **error)
+{
+	GsApp *app;
+	g_autoptr(GsApp) app_new = gs_app_new (NULL);
+
+	/* refine enough to get the unique ID */
+	if (!gs_appstream_refine_app (plugin, app_new, silo, component,
+				      GS_PLUGIN_REFINE_FLAGS_DEFAULT,
+				      error))
+		return NULL;
+
+	/* never add wildcard apps to the plugin cache */
+	if (gs_app_has_quirk (app_new, GS_APP_QUIRK_IS_WILDCARD))
+		return g_steal_pointer (&app_new);
+
+	/* no longer supported */
+	if (gs_app_get_kind (app_new) == AS_APP_KIND_SHELL_EXTENSION) {
+		g_set_error (error,
+			     GS_PLUGIN_ERROR,
+			     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+			     "shell extensions no longer supported");
+		return NULL;
+	}
+
+	/* look for existing object */
+	app = gs_plugin_cache_lookup (plugin, gs_app_get_unique_id (app_new));
+	if (app != NULL)
+		return app;
+
+	/* use the temp object we just created */
+	gs_app_set_metadata (app_new, "GnomeSoftware::Creator",
+			     gs_plugin_get_name (plugin));
+	gs_plugin_cache_add (plugin, NULL, app_new);
+	return g_steal_pointer (&app_new);
+}
+
+static gchar *
+gs_appstream_format_description (XbNode *root, GError **error)
+{
+	g_autoptr(GString) str = g_string_new (NULL);
+	g_autoptr(XbNode) n = xb_node_get_child (root);
+
+	while (n != NULL) {
+		g_autoptr(XbNode) n2 = NULL;
+
+		/* support <p>, <ul>, <ol> and <li>, ignore all else */
+		if (g_strcmp0 (xb_node_get_element (n), "p") == 0) {
+			g_string_append_printf (str, "%s\n\n", xb_node_get_text (n));
+		} else if (g_strcmp0 (xb_node_get_element (n), "ul") == 0) {
+			g_autoptr(GPtrArray) children = xb_node_get_children (n);
+			for (guint i = 0; i < children->len; i++) {
+				XbNode *nc = g_ptr_array_index (children, i);
+				if (g_strcmp0 (xb_node_get_element (nc), "li") == 0) {
+					g_string_append_printf (str, " • %s\n",
+								xb_node_get_text (nc));
+				}
+			}
+			g_string_append (str, "\n");
+		} else if (g_strcmp0 (xb_node_get_element (n), "ol") == 0) {
+			g_autoptr(GPtrArray) children = xb_node_get_children (n);
+			for (guint i = 0; i < children->len; i++) {
+				XbNode *nc = g_ptr_array_index (children, i);
+				if (g_strcmp0 (xb_node_get_element (nc), "li") == 0) {
+					g_string_append_printf (str, " %u. %s\n",
+								i + 1,
+								xb_node_get_text (nc));
+				}
+			}
+			g_string_append (str, "\n");
+		}
+
+		n2 = xb_node_get_next (n);
+		g_set_object (&n, n2);
+	}
+
+	/* remove extra newlines */
+	while (str->len > 0 && str->str[str->len - 1] == '\n')
+		g_string_truncate (str, str->len - 1);
+
+	/* success */
+	return g_string_free (g_steal_pointer (&str), FALSE);
+}
+
+static gchar *
+gs_appstream_build_icon_prefix (XbNode *component)
+{
+	const gchar *origin;
+	const gchar *tmp;
+	gint npath;
+	g_auto(GStrv) path = NULL;
+	g_autoptr(XbNode) components = NULL;
+
+	/* no parent, e.g. AppData */
+	components = xb_node_get_parent (component);
+	if (components == NULL)
+		return NULL;
+
+	/* set explicitly */
+	tmp = xb_node_query_text (components, "info/icon-prefix", NULL);
+	if (tmp != NULL)
+		return g_strdup (tmp);
+
+	/* fall back to origin */
+	origin = xb_node_get_attr (components, "origin");
+	if (origin == NULL)
+		return NULL;
+
+	/* no metadata */
+	tmp = xb_node_query_text (components, "info/filename", NULL);
+	if (tmp == NULL)
+		return NULL;
+
+	/* check format */
+	path = g_strsplit (tmp, "/", -1);
+	npath = g_strv_length (path);
+	if (npath < 3 || !(g_strcmp0 (path[npath-2], "xmls") == 0 || g_strcmp0 (path[npath-2], "yaml") == 0))
+		return NULL;
+
+	/* fix the new path */
+	g_free (path[npath-1]);
+	g_free (path[npath-2]);
+	path[npath-1] = g_strdup (origin);
+	path[npath-2] = g_strdup ("icons");
+	return g_strjoinv ("/", path);
+}
+
+static AsIcon *
+gs_appstream_new_icon (XbNode *component, XbNode *n, AsIconKind icon_kind, guint sz)
+{
+	AsIcon *icon = as_icon_new ();
+	g_autofree gchar *icon_path = NULL;
+	as_icon_set_kind (icon, icon_kind);
+	switch (icon_kind) {
+	case AS_ICON_KIND_REMOTE:
+		as_icon_set_url (icon, xb_node_get_text (n));
+		break;
+	default:
+		as_icon_set_name (icon, xb_node_get_text (n));
+	}
+	if (sz == 0)
+		sz = xb_node_get_attr_as_uint (n, "width");
+	if (sz > 0) {
+		as_icon_set_width (icon, sz);
+		as_icon_set_height (icon, sz);
+	}
+	icon_path = gs_appstream_build_icon_prefix (component);
+	if (icon_path != NULL)
+		as_icon_set_prefix (icon, icon_path);
+	return icon;
+}
+
+static AsIcon *
+gs_appstream_get_icon_by_kind (XbNode *component, AsIconKind icon_kind)
+{
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(XbNode) icon = NULL;
+
+	xpath = g_strdup_printf ("icon[@type='%s']",
+				 as_icon_kind_to_string (icon_kind));
+	icon = xb_node_query_first (component, xpath, NULL);
+	if (icon == NULL)
+		return NULL;
+	return gs_appstream_new_icon (component, icon, icon_kind, 0);
+}
+
+static AsIcon *
+gs_appstream_get_icon_by_kind_and_size (XbNode *component, AsIconKind icon_kind, guint sz)
+{
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(XbNode) icon = NULL;
+
+	xpath = g_strdup_printf ("icon[@type='%s'][@height='%u'][@width='%u']",
+				 as_icon_kind_to_string (icon_kind), sz, sz);
+	icon = xb_node_query_first (component, xpath, NULL);
+	if (icon == NULL)
+		return NULL;
+	return gs_appstream_new_icon (component, icon, icon_kind, sz);
+}
+
+static void
+gs_appstream_refine_icon (GsPlugin *plugin, GsApp *app, XbNode *component)
+{
+	g_autoptr(AsIcon) icon = NULL;
+	g_autoptr(XbNode) n = NULL;
+
+	/* try a stock icon first */
+	icon = gs_appstream_get_icon_by_kind (component, AS_ICON_KIND_STOCK);
+	if (icon != NULL) {
+		/* the stock icon referenced by the AppStream data may not be present in the current
+		 * theme (usually more stock icon entries are added to permit huge themes like Papirus
+		 * to style all apps in the software center). Since we can not rely on the icon's presence,
+		 * we also add other icons to the list and do not return here. */
+		gs_app_add_icon (app, icon);
+	}
+
+	/* if HiDPI get a 128px cached icon */
+	if (gs_plugin_get_scale (plugin) == 2) {
+		icon = gs_appstream_get_icon_by_kind_and_size (component,
+							       AS_ICON_KIND_CACHED,
+							       128);
+		if (icon != NULL) {
+			gs_app_add_icon (app, icon);
+			return;
+		}
+	}
+
+	/* non-HiDPI cached icon */
+	icon = gs_appstream_get_icon_by_kind_and_size (component,
+						       AS_ICON_KIND_CACHED,
+						       64);
+	if (icon != NULL) {
+		gs_app_add_icon (app, icon);
+		return;
+	}
+
+	/* prefer local */
+	icon = gs_appstream_get_icon_by_kind (component, AS_ICON_KIND_LOCAL);
+	if (icon != NULL) {
+		/* does not exist, so try to find using the icon theme */
+		if (as_icon_get_kind (icon) == AS_ICON_KIND_LOCAL &&
+		    as_icon_get_filename (icon) == NULL) {
+			g_debug ("converting missing LOCAL icon %s to STOCK",
+				 as_icon_get_name (icon));
+			as_icon_set_kind (icon, AS_ICON_KIND_STOCK);
+		}
+		gs_app_add_icon (app, icon);
+		return;
+	}
+
+	/* remote URL */
+	icon = gs_appstream_get_icon_by_kind (component, AS_ICON_KIND_REMOTE);
+	if (icon != NULL) {
+		gs_app_add_icon (app, icon);
+		return;
+	}
+
+	/* assume a stock icon */
+	n = xb_node_query_first (component, "icon", NULL);
+	if (n != NULL) {
+		icon = gs_appstream_new_icon (component, n, AS_ICON_KIND_STOCK, 0);
+		gs_app_add_icon (app, icon);
+	}
+}
+
+static gboolean
+gs_appstream_refine_add_addons (GsPlugin *plugin,
+				GsApp *app,
+				XbSilo *silo,
+				GError **error)
+{
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) addons = NULL;
+
+	/* get all components */
+	xpath = g_strdup_printf ("components/component/extends[text()='%s']/..",
+				 gs_app_get_id (app));
+	addons = xb_silo_query (silo, xpath, 0, &error_local);
+	if (addons == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < addons->len; i++) {
+		XbNode *addon = g_ptr_array_index (addons, i);
+		g_autoptr(GsApp) app2 = NULL;
+		app2 = gs_appstream_create_app (plugin, silo, addon, error);
+		if (app2 == NULL)
+			return FALSE;
+		gs_app_add_addon (app, app2);
+	}
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_refine_add_images (GsApp *app, AsScreenshot *ss, XbNode *screenshot, GError **error)
+{
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) images = NULL;
+
+	/* get all components */
+	images = xb_node_query (screenshot, "image", 0, &error_local);
+	if (images == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < images->len; i++) {
+		XbNode *image = g_ptr_array_index (images, i);
+		g_autoptr(AsImage) im = as_image_new ();
+		as_image_set_height (im, xb_node_get_attr_as_uint (image, "height"));
+		as_image_set_width (im, xb_node_get_attr_as_uint (image, "width"));
+		as_image_set_kind (im, as_image_kind_from_string (xb_node_get_attr (image, "type")));
+		as_image_set_url (im, xb_node_get_text (image));
+		as_screenshot_add_image (ss, im);
+	}
+
+	/* success */
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_refine_add_screenshots (GsApp *app, XbNode *component, GError **error)
+{
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) screenshots = NULL;
+
+	/* get all components */
+	screenshots = xb_node_query (component, "screenshots/screenshot", 0, &error_local);
+	if (screenshots == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < screenshots->len; i++) {
+		XbNode *screenshot = g_ptr_array_index (screenshots, i);
+		g_autoptr(AsScreenshot) ss = as_screenshot_new ();
+		if (!gs_appstream_refine_add_images (app, ss, screenshot, error))
+			return FALSE;
+		gs_app_add_screenshot (app, ss);
+	}
+
+	/* FIXME: move into no refine flags section? */
+	if (screenshots ->len > 0)
+		gs_app_add_kudo (app, GS_APP_KUDO_HAS_SCREENSHOTS);
+
+	/* success */
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_refine_add_provides (GsApp *app, XbNode *component, GError **error)
+{
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) provides = NULL;
+
+	/* get all components */
+	provides = xb_node_query (component, "provides/*", 0, &error_local);
+	if (provides == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < provides->len; i++) {
+		XbNode *provide = g_ptr_array_index (provides, i);
+		g_autoptr(AsProvide) pr = as_provide_new ();
+		as_provide_set_kind (pr, as_provide_kind_from_string (xb_node_get_element (provide)));
+		as_provide_set_value (pr, xb_node_get_text (provide));
+		gs_app_add_provide (app, pr);
+	}
+
+	/* success */
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_is_recent_release (XbNode *component)
+{
+	guint64 ts;
+	guint64 secs;
+
+	/* get newest release */
+	ts = xb_node_query_attr_as_uint (component, "releases/release", "timestamp", NULL);
+	if (ts == G_MAXUINT64)
+		return FALSE;
+
+	/* is last build less than one year ago? */
+	secs = ((guint64) g_get_real_time () / G_USEC_PER_SEC) - ts;
+	return secs / (60 * 60 * 24) < 365;
+}
+
+static gboolean
+gs_appstream_copy_metadata (GsApp *app, XbNode *component, GError **error)
+{
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) values = NULL;
+
+	/* get all components */
+	values = xb_node_query (component, "custom/value", 0, &error_local);
+	if (values == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < values->len; i++) {
+		XbNode *value = g_ptr_array_index (values, i);
+		const gchar *key = xb_node_get_attr (value, "key");
+		if (key == NULL)
+			continue;
+		if (gs_app_get_metadata_item (app, key) != NULL)
+			continue;
+		gs_app_set_metadata (app, key, xb_node_get_text (value));
+	}
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_refine_app_updates (GsPlugin *plugin,
+				 GsApp *app,
+				 XbSilo *silo,
+				 XbNode *component,
+				 GError **error)
+{
+	AsUrgencyKind urgency_best = AS_URGENCY_KIND_UNKNOWN;
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GHashTable) installed = g_hash_table_new (g_str_hash, g_str_equal);
+	g_autoptr(GPtrArray) releases_inst = NULL;
+	g_autoptr(GPtrArray) releases = NULL;
+	g_autoptr(GPtrArray) updates_list = g_ptr_array_new ();
+
+	/* only for UPDATABLE apps */
+	if (!gs_app_is_updatable (app))
+		return TRUE;
+
+	/* find out which releases are already installed */
+	xpath = g_strdup_printf ("component/id[text()='%s']/../releases/*[@version]",
+				 gs_app_get_id (app));
+	releases_inst = xb_silo_query (silo, xpath, 0, &error_local);
+	if (releases_inst == NULL) {
+		if (!g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND) &&
+		    !g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT)) {
+			g_propagate_error (error, g_steal_pointer (&error_local));
+			return FALSE;
+		}
+	} else {
+		for (guint i = 0; i < releases_inst->len; i++) {
+			XbNode *release = g_ptr_array_index (releases_inst, i);
+			g_hash_table_insert (installed,
+					     (gpointer) xb_node_get_attr (release, "version"),
+					     (gpointer) release);
+		}
+	}
+	g_clear_error (&error_local);
+
+	/* get all components */
+	releases = xb_node_query (component, "releases/*", 0, &error_local);
+	if (releases == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < releases->len; i++) {
+		XbNode *release = g_ptr_array_index (releases, i);
+		const gchar *version = xb_node_get_attr (release, "version");
+		g_autoptr(XbNode) description = NULL;
+		AsUrgencyKind urgency_tmp;
+
+		/* ignore releases with no version */
+		if (version == NULL)
+			continue;
+
+		/* already installed */
+		if (g_hash_table_lookup (installed, version) != NULL)
+			continue;
+
+		/* limit this to three versions backwards if there has never
+		 * been a detected installed version */
+		if (g_hash_table_size (installed) == 0 && i >= 3)
+			break;
+
+		/* use the 'worst' urgency, e.g. critical over enhancement */
+		urgency_tmp = as_urgency_kind_from_string (xb_node_get_attr (release, "urgency"));
+		if (urgency_tmp > urgency_best)
+			urgency_best = urgency_tmp;
+
+		/* add updates with a description */
+		description = xb_node_query_first (release, "description", NULL);
+		if (description == NULL)
+			continue;
+		g_ptr_array_add (updates_list, release);
+	}
+
+	/* only set if known */
+	if (urgency_best != AS_URGENCY_KIND_UNKNOWN)
+		gs_app_set_update_urgency (app, urgency_best);
+
+	/* no prefix on each release */
+	if (updates_list->len == 1) {
+		XbNode *release = g_ptr_array_index (updates_list, 0);
+		g_autoptr(XbNode) n = NULL;
+		g_autofree gchar *desc = NULL;
+		n = xb_node_query_first (release, "description", NULL);
+		desc = gs_appstream_format_description (n, NULL);
+		gs_app_set_update_details (app, desc);
+
+	/* get the descriptions with a version prefix */
+	} else if (updates_list->len > 1) {
+		g_autoptr(GString) update_desc = g_string_new ("");
+		for (guint i = 0; i < updates_list->len; i++) {
+			XbNode *release = g_ptr_array_index (updates_list, i);
+			g_autofree gchar *desc = NULL;
+			g_autoptr(XbNode) n = NULL;
+
+			n = xb_node_query_first (release, "description", NULL);
+			desc = gs_appstream_format_description (n, NULL);
+			g_string_append_printf (update_desc,
+						"Version %s:\n%s\n\n",
+						xb_node_get_attr (release, "version"),
+						desc);
+		}
+
+		/* remove trailing newlines */
+		if (update_desc->len > 2)
+			g_string_truncate (update_desc, update_desc->len - 2);
+		gs_app_set_update_details (app, update_desc->str);
+	}
+
+	/* if there is no already set update version use the newest */
+	if (gs_app_get_update_version (app) == NULL &&
+	    updates_list->len > 0) {
+		XbNode *release = g_ptr_array_index (updates_list, 0);
+		gs_app_set_update_version (app, xb_node_get_attr (release, "version"));
+	}
+
+	/* success */
+	return TRUE;
+}
+
+/**
+ * _gs_utils_locale_has_translations:
+ * @locale: A locale, e.g. `en_GB` or `uz_UZ.utf8@cyrillic`
+ *
+ * Looks up if the locale is likely to have translations.
+ *
+ * Returns: %TRUE if the locale should have translations
+ **/
+static gboolean
+_gs_utils_locale_has_translations (const gchar *locale)
+{
+	g_autofree gchar *locale_copy = g_strdup (locale);
+	gchar *separator;
+
+	/* Strip off the codeset and modifier, if present. */
+	separator = strpbrk (locale_copy, ".@");
+	if (separator != NULL)
+		*separator = '\0';
+
+	if (g_strcmp0 (locale_copy, "C") == 0)
+		return FALSE;
+	if (g_strcmp0 (locale_copy, "en") == 0)
+		return FALSE;
+	if (g_strcmp0 (locale_copy, "en_US") == 0)
+		return FALSE;
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_origin_valid (const gchar *origin)
+{
+	if (origin == NULL)
+		return FALSE;
+	if (g_strcmp0 (origin, "") == 0)
+		return FALSE;
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_is_valid_project_group (const gchar *project_group)
+{
+	if (project_group == NULL)
+		return FALSE;
+	return as_utils_is_environment_id (project_group);
+}
+
+static gboolean
+gs_appstream_refine_app_content_rating (GsPlugin *plugin,
+					GsApp *app,
+					XbNode *content_rating,
+					GError **error)
+{
+	g_autoptr(AsContentRating) cr = as_content_rating_new ();
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) content_attributes = NULL;
+	const gchar *content_rating_kind = NULL;
+
+	/* get kind */
+	content_rating_kind = xb_node_get_attr (content_rating, "type");
+	/* we only really expect/support OARS 1.0 and 1.1 */
+	if (content_rating_kind == NULL ||
+	    (g_strcmp0 (content_rating_kind, "oars-1.0") != 0 &&
+	     g_strcmp0 (content_rating_kind, "oars-1.1") != 0)) {
+		return TRUE;
+	}
+
+	as_content_rating_set_kind (cr, content_rating_kind);
+
+	/* get attributes; no attributes being found (i.e.
+	 * `<content_rating type="*"/>`) is OK: it means that all attributes have
+	 * value `none`, as per the
+	 * [OARS semantics](https://github.com/hughsie/oars/blob/master/specification/oars-1.1.md) */
+	content_attributes = xb_node_query (content_rating, "content_attribute", 0, &error_local);
+	if (content_attributes == NULL &&
+	    g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) {
+		g_clear_error (&error_local);
+	} else if (content_attributes == NULL &&
+		 g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT)) {
+		return TRUE;
+	} else if (content_attributes == NULL) {
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+
+	for (guint i = 0; content_attributes != NULL && i < content_attributes->len; i++) {
+		XbNode *content_attribute = g_ptr_array_index (content_attributes, i);
+		as_content_rating_add_attribute (cr,
+						 xb_node_get_attr (content_attribute, "id"),
+						 as_content_rating_value_from_string (xb_node_get_text (content_attribute)));
+	}
+
+	gs_app_set_content_rating (app, cr);
+	return TRUE;
+}
+
+static gboolean
+gs_appstream_refine_app_content_ratings (GsPlugin *plugin,
+					 GsApp *app,
+					 XbNode *component,
+					 GError **error)
+{
+	g_autoptr(GPtrArray) content_ratings = NULL;
+	g_autoptr(GError) error_local = NULL;
+
+	/* find any content ratings */
+	content_ratings = xb_node_query (component, "content_rating", 0, &error_local);
+	if (content_ratings == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < content_ratings->len; i++) {
+		XbNode *content_rating = g_ptr_array_index (content_ratings, i);
+		if (!gs_appstream_refine_app_content_rating (plugin, app, content_rating, error))
+			return FALSE;
+	}
+	return TRUE;
+}
+
+gboolean
+gs_appstream_refine_app (GsPlugin *plugin,
+			 GsApp *app,
+			 XbSilo *silo,
+			 XbNode *component,
+			 GsPluginRefineFlags refine_flags,
+			 GError **error)
+{
+	const gchar *tmp;
+	g_autoptr(GPtrArray) bundles = NULL;
+	g_autoptr(GPtrArray) launchables = NULL;
+	g_autoptr(XbNode) req = NULL;
+
+	/* is compatible */
+	req = xb_node_query_first (component,
+				   "requires/id[@type='id']"
+				   "[text()='org.gnome.Software.desktop']", NULL);
+	if (req != NULL) {
+#if AS_CHECK_VERSION(0,7,15)
+		gint rc = as_utils_vercmp_full (xb_node_get_attr (req, "version"),
+		                                PACKAGE_VERSION,
+		                                AS_VERSION_COMPARE_FLAG_NONE);
+#else
+		gint rc = as_utils_vercmp (xb_node_get_attr (req, "version"),
+		                           PACKAGE_VERSION);
+#endif
+		if (rc > 0) {
+			g_set_error (error,
+				     GS_PLUGIN_ERROR,
+				     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+				     "not for this gnome-software");
+			return FALSE;
+		}
+	}
+
+	/* types we can never launch */
+	switch (gs_app_get_kind (app)) {
+	case AS_APP_KIND_ADDON:
+	case AS_APP_KIND_CODEC:
+	case AS_APP_KIND_DRIVER:
+	case AS_APP_KIND_FIRMWARE:
+	case AS_APP_KIND_FONT:
+	case AS_APP_KIND_GENERIC:
+	case AS_APP_KIND_INPUT_METHOD:
+	case AS_APP_KIND_LOCALIZATION:
+	case AS_APP_KIND_OS_UPDATE:
+	case AS_APP_KIND_OS_UPGRADE:
+	case AS_APP_KIND_RUNTIME:
+	case AS_APP_KIND_SOURCE:
+		gs_app_add_quirk (app, GS_APP_QUIRK_NOT_LAUNCHABLE);
+		break;
+	default:
+		break;
+	}
+
+	/* check if the special metadata affects the not-launchable quirk */
+	tmp = gs_app_get_metadata_item (app, "GnomeSoftware::quirks::not-launchable");
+	if (tmp != NULL) {
+		if (g_strcmp0 (tmp, "true") == 0)
+			gs_app_add_quirk (app, GS_APP_QUIRK_NOT_LAUNCHABLE);
+		else if (g_strcmp0 (tmp, "false") == 0)
+			gs_app_remove_quirk (app, GS_APP_QUIRK_NOT_LAUNCHABLE);
+	}
+
+	tmp = gs_app_get_metadata_item (app, "GnomeSoftware::quirks::hide-everywhere");
+	if (tmp != NULL) {
+		if (g_strcmp0 (tmp, "true") == 0)
+			gs_app_add_quirk (app, GS_APP_QUIRK_HIDE_EVERYWHERE);
+		else if (g_strcmp0 (tmp, "false") == 0)
+			gs_app_remove_quirk (app, GS_APP_QUIRK_HIDE_EVERYWHERE);
+	}
+
+	/* try to detect old-style AppStream 'override'
+	 * files without the merge attribute */
+	if (xb_node_query_text (component, "name", NULL) == NULL &&
+	    xb_node_query_text (component, "metadata_license", NULL) == NULL) {
+		gs_app_add_quirk (app, GS_APP_QUIRK_IS_WILDCARD);
+	}
+
+	/* set id */
+	tmp = xb_node_query_text (component, "id", NULL);
+	if (tmp != NULL && gs_app_get_id (app) == NULL)
+		gs_app_set_id (app, tmp);
+
+	/* set source */
+	tmp = xb_node_query_text (component, "../info/filename", NULL);
+	if (tmp != NULL && gs_app_get_metadata_item (app, "appstream::source-file") == NULL) {
+		gs_app_set_metadata (app, "appstream::source-file", tmp);
+	}
+
+	/* set scope */
+	tmp = xb_node_query_text (component, "../info/scope", NULL);
+	if (tmp != NULL)
+		gs_app_set_scope (app, as_app_scope_from_string (tmp));
+
+	/* set content rating */
+	if (TRUE) {
+		if (!gs_appstream_refine_app_content_ratings (plugin, app, component, error))
+			return FALSE;
+	}
+
+	/* set name */
+	tmp = xb_node_query_text (component, "name", NULL);
+	if (tmp != NULL)
+		gs_app_set_name (app, GS_APP_QUALITY_HIGHEST, tmp);
+
+	/* set summary */
+	tmp = xb_node_query_text (component, "summary", NULL);
+	if (tmp != NULL)
+		gs_app_set_summary (app, GS_APP_QUALITY_HIGHEST, tmp);
+
+	/* add urls */
+	if (refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_URL) {
+		g_autoptr(GPtrArray) urls = NULL;
+		urls = xb_node_query (component, "url", 0, NULL);
+		if (urls != NULL) {
+			for (guint i = 0; i < urls->len; i++) {
+				XbNode *url = g_ptr_array_index (urls, i);
+				const gchar *kind = xb_node_get_attr (url, "type");
+				if (kind == NULL)
+					continue;
+				gs_app_set_url (app,
+						as_url_kind_from_string (kind),
+						xb_node_get_text (url));
+			}
+		}
+	}
+
+	/* add launchables */
+	launchables = xb_node_query (component, "launchable", 0, NULL);
+	if (launchables != NULL) {
+		for (guint i = 0; i < launchables->len; i++) {
+			XbNode *launchable = g_ptr_array_index (launchables, i);
+			const gchar *kind = xb_node_get_attr (launchable, "type");
+			if (g_strcmp0 (kind, "desktop-id") == 0) {
+				gs_app_set_launchable (app,
+						       AS_LAUNCHABLE_KIND_DESKTOP_ID,
+						       xb_node_get_text (launchable));
+				break;
+			} else if (g_strcmp0 (kind, "url") == 0) {
+				gs_app_set_launchable (app,
+						       AS_LAUNCHABLE_KIND_URL,
+						       xb_node_get_text (launchable));
+			}
+		}
+	}
+
+	/* set license */
+	if ((refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_LICENSE) > 0 &&
+	    gs_app_get_license (app) == NULL) {
+		tmp = xb_node_query_text (component, "project_license", NULL);
+		if (tmp != NULL)
+			gs_app_set_license (app, GS_APP_QUALITY_HIGHEST, tmp);
+	}
+
+	/* set description */
+	if (refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_DESCRIPTION) {
+		g_autofree gchar *description = NULL;
+		g_autoptr(XbNode) n = xb_node_query_first (component, "description", NULL);
+		if (n != NULL)
+			description = gs_appstream_format_description (n, NULL);
+		if (description != NULL)
+			gs_app_set_description (app, GS_APP_QUALITY_HIGHEST, description);
+	}
+
+	/* set icon */
+	if ((refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_ICON) > 0 &&
+	    gs_app_get_icons(app)->len == 0)
+		gs_appstream_refine_icon (plugin, app, component);
+
+	/* set categories */
+	if (refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_CATEGORIES) {
+		g_autoptr(GPtrArray) categories = NULL;
+		categories = xb_node_query (component, "categories/category", 0, NULL);
+		if (categories != NULL) {
+			for (guint i = 0; i < categories->len; i++) {
+				XbNode *category = g_ptr_array_index (categories, i);
+				gs_app_add_category (app, xb_node_get_text (category));
+
+				/* Special case: We used to use the `Blacklisted`
+				 * category to hide apps from their .desktop
+				 * file or appdata. We now use a quirk for that.
+				 * This special case can be removed when all
+				 * appstream files no longer use the `Blacklisted`
+				 * category (including external-appstream files
+				 * put together by distributions). */
+				if (g_strcmp0 (xb_node_get_text (category), "Blacklisted") == 0)
+					gs_app_add_quirk (app, GS_APP_QUIRK_HIDE_EVERYWHERE);
+			}
+		}
+	}
+
+	/* set project group */
+	if ((refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_PROJECT_GROUP) > 0 &&
+	    gs_app_get_project_group (app) == NULL) {
+		tmp = xb_node_query_text (component, "project_group", NULL);
+		if (tmp != NULL && gs_appstream_is_valid_project_group (tmp))
+			gs_app_set_project_group (app, tmp);
+	}
+
+	/* set developer name */
+	if ((refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_DEVELOPER_NAME) > 0 &&
+	    gs_app_get_developer_name (app) == NULL) {
+		tmp = xb_node_query_text (component, "developer_name", NULL);
+		if (tmp != NULL)
+			gs_app_set_developer_name (app, tmp);
+	}
+
+	/* set id kind */
+	if (gs_app_get_kind (app) == AS_APP_KIND_UNKNOWN ||
+	    gs_app_get_kind (app) == AS_APP_KIND_GENERIC) {
+		tmp = xb_node_get_attr (component, "type");
+		gs_app_set_kind (app, as_app_kind_from_string (tmp));
+	}
+
+	/* copy all the metadata */
+	if (!gs_appstream_copy_metadata (app, component, error))
+		return FALSE;
+
+	/* add bundles */
+	bundles = xb_node_query (component, "bundle", 0, NULL);
+	if (bundles != NULL && gs_app_get_sources(app)->len == 0) {
+		for (guint i = 0; i < bundles->len; i++) {
+			XbNode *bundle = g_ptr_array_index (bundles, i);
+			const gchar *kind = xb_node_get_attr (bundle, "type");
+			gs_app_add_source (app, xb_node_get_text (bundle));
+			gs_app_set_bundle_kind (app, as_bundle_kind_from_string (kind));
+
+			/* get the type/name/arch/branch */
+			if (gs_app_get_bundle_kind (app) == AS_BUNDLE_KIND_FLATPAK) {
+				g_auto(GStrv) split = g_strsplit (xb_node_get_text (bundle), "/", -1);
+				if (g_strv_length (split) != 4) {
+					g_set_error (error,
+						     GS_PLUGIN_ERROR,
+						     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+						     "invalid ID %s for a flatpak ref",
+						     xb_node_get_text (bundle));
+					return FALSE;
+				}
+
+				/* we only need the branch for the unique ID */
+				gs_app_set_branch (app, split[3]);
+			}
+		}
+	}
+
+	/* add legacy package names */
+	if (gs_app_get_bundle_kind (app) == AS_BUNDLE_KIND_UNKNOWN) {
+		g_autoptr(GPtrArray) pkgnames = NULL;
+		pkgnames = xb_node_query (component, "pkgname", 0, NULL);
+		if (pkgnames != NULL && gs_app_get_sources(app)->len == 0) {
+			for (guint i = 0; i < pkgnames->len; i++) {
+				XbNode *pkgname = g_ptr_array_index (pkgnames, i);
+				tmp = xb_node_get_text (pkgname);
+				if (tmp != NULL && tmp[0] != '\0')
+					gs_app_add_source (app, tmp);
+			}
+			gs_app_set_bundle_kind (app, AS_BUNDLE_KIND_PACKAGE);
+		}
+	}
+
+	/* set origin for flatpaks */
+	if (gs_app_get_origin (app) == NULL &&
+	    gs_app_get_bundle_kind (app) == AS_BUNDLE_KIND_FLATPAK) {
+		g_autoptr(XbNode) parent = xb_node_get_parent (component);
+		if (parent != NULL) {
+			tmp = xb_node_get_attr (parent, "origin");
+			gs_app_set_origin (app, tmp);
+		}
+	}
+
+	/* set addons */
+	if (refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_ADDONS) {
+		if (!gs_appstream_refine_add_addons (plugin, app, silo, error))
+			return FALSE;
+	}
+
+	/* set screenshots */
+	if ((refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_SCREENSHOTS) > 0 &&
+	    gs_app_get_screenshots(app)->len == 0) {
+		if (!gs_appstream_refine_add_screenshots (app, component, error))
+			return FALSE;
+	}
+
+	/* set provides */
+	if (!gs_appstream_refine_add_provides (app, component, error))
+		return FALSE;
+
+	/* add kudos */
+	if (refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_KUDOS) {
+		g_autoptr(GPtrArray) kudos = NULL;
+		tmp = gs_plugin_get_locale (plugin);
+		if (!_gs_utils_locale_has_translations (tmp)) {
+			gs_app_add_kudo (app, GS_APP_KUDO_MY_LANGUAGE);
+		} else {
+
+			g_autoptr(GString) xpath = g_string_new (NULL);
+			g_auto(GStrv) variants = g_get_locale_variants (tmp);
+
+			/* @variants includes @tmp */
+			for (gsize i = 0; variants[i] != NULL; i++)
+				xb_string_append_union (xpath, "languages/lang[text()='%s'][@percentage>50]", variants[i]);
+
+			if (xb_node_query_text (component, xpath->str, NULL) != NULL)
+				gs_app_add_kudo (app, GS_APP_KUDO_MY_LANGUAGE);
+		}
+
+		/* any keywords */
+		if (xb_node_query_text (component, "keywords/keyword", NULL) != NULL)
+			gs_app_add_kudo (app, GS_APP_KUDO_HAS_KEYWORDS);
+
+		/* HiDPI icon */
+		if (xb_node_query_text (component, "icon[@width='128']", NULL) != NULL)
+			gs_app_add_kudo (app, GS_APP_KUDO_HI_DPI_ICON);
+
+		/* was this application released recently */
+		if (gs_appstream_is_recent_release (component))
+			gs_app_add_kudo (app, GS_APP_KUDO_RECENT_RELEASE);
+
+		/* add a kudo to featured and popular apps */
+		if (xb_node_query_text (component, "kudos/kudo[text()='GnomeSoftware::popular']", NULL) != NULL)
+			gs_app_add_kudo (app, GS_APP_KUDO_FEATURED_RECOMMENDED);
+		if (xb_node_query_text (component, "categories/category[text()='Featured']", NULL) != NULL)
+			gs_app_add_kudo (app, GS_APP_KUDO_FEATURED_RECOMMENDED);
+
+		/* add new-style kudos */
+		kudos = xb_node_query (component, "kudos/kudo", 0, NULL);
+		for (guint i = 0; kudos != NULL && i < kudos->len; i++) {
+			XbNode *kudo = g_ptr_array_index (kudos, i);
+			switch (as_kudo_kind_from_string (xb_node_get_text (kudo))) {
+			case AS_KUDO_KIND_SEARCH_PROVIDER:
+				gs_app_add_kudo (app, GS_APP_KUDO_SEARCH_PROVIDER);
+				break;
+			case AS_KUDO_KIND_USER_DOCS:
+				gs_app_add_kudo (app, GS_APP_KUDO_INSTALLS_USER_DOCS);
+				break;
+			case AS_KUDO_KIND_MODERN_TOOLKIT:
+				gs_app_add_kudo (app, GS_APP_KUDO_MODERN_TOOLKIT);
+				break;
+			case AS_KUDO_KIND_NOTIFICATIONS:
+				gs_app_add_kudo (app, GS_APP_KUDO_USES_NOTIFICATIONS);
+				break;
+			case AS_KUDO_KIND_HIGH_CONTRAST:
+				gs_app_add_kudo (app, GS_APP_KUDO_HIGH_CONTRAST);
+				break;
+			case AS_KUDO_KIND_HI_DPI_ICON:
+				gs_app_add_kudo (app, GS_APP_KUDO_HI_DPI_ICON);
+				break;
+			default:
+				break;
+			}
+		}
+	}
+
+	/* we have an origin in the XML */
+	if ((refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_ORIGIN) > 0 &&
+	    gs_app_get_origin_appstream (app) == NULL) {
+		g_autoptr(XbNode) parent = xb_node_get_parent (component);
+		if (parent != NULL) {
+			tmp = xb_node_get_attr (parent, "origin");
+			if (gs_appstream_origin_valid (tmp))
+				gs_app_set_origin_appstream (app, tmp);
+		}
+	}
+
+	/* is there any update information */
+	if (refine_flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_UPDATE_DETAILS) {
+		if (!gs_appstream_refine_app_updates (plugin,
+						      app,
+						      silo,
+						      component,
+						      error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
+
+typedef struct {
+	AsAppSearchMatch	 match_value;
+	XbQuery			*query;
+} GsAppstreamSearchHelper;
+
+static void
+gs_appstream_search_helper_free (GsAppstreamSearchHelper *helper)
+{
+	g_object_unref (helper->query);
+	g_free (helper);
+}
+
+static guint16
+gs_appstream_silo_search_component2 (GPtrArray *array, XbNode *component, const gchar *search)
+{
+	guint16 match_value = 0;
+
+	/* do searches */
+	for (guint i = 0; i < array->len; i++) {
+		g_autoptr(GPtrArray) n = NULL;
+		GsAppstreamSearchHelper *helper = g_ptr_array_index (array, i);
+		xb_query_bind_str (helper->query, 0, search, NULL);
+		n = xb_node_query_full (component, helper->query, NULL);
+		if (n != NULL)
+			match_value |= helper->match_value;
+	}
+	return match_value;
+}
+
+static guint16
+gs_appstream_silo_search_component (GPtrArray *array, XbNode *component, const gchar * const *search)
+{
+	guint16 matches_sum = 0;
+
+	/* do *all* search keywords match */
+	for (guint i = 0; search[i] != NULL; i++) {
+		guint tmp = gs_appstream_silo_search_component2 (array, component, search[i]);
+		if (tmp == 0)
+			return 0;
+		matches_sum |= tmp;
+	}
+	return matches_sum;
+}
+
+gboolean
+gs_appstream_search (GsPlugin *plugin,
+		     XbSilo *silo,
+		     const gchar * const *values,
+		     GsAppList *list,
+		     GCancellable *cancellable,
+		     GError **error)
+{
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) array = g_ptr_array_new_with_free_func ((GDestroyNotify) gs_appstream_search_helper_free);
+	g_autoptr(GPtrArray) components = NULL;
+	g_autoptr(GTimer) timer = g_timer_new ();
+	const struct {
+		AsAppSearchMatch	 match_value;
+		const gchar		*xpath;
+	} queries[] = {
+		{ AS_APP_SEARCH_MATCH_MIMETYPE,	"mimetypes/mimetype[text()~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_PKGNAME,	"pkgname[text()~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_COMMENT,	"summary[text()~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_NAME,	"name[text()~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_KEYWORD,	"keywords/keyword[text()~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_ID,	"id[text()~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_ID,	"launchable[text()~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_ORIGIN,	"../components[@origin~=stem(?)]" },
+		{ AS_APP_SEARCH_MATCH_NONE,	NULL }
+	};
+
+	/* add some weighted queries */
+	for (guint i = 0; queries[i].xpath != NULL; i++) {
+		g_autoptr(GError) error_query = NULL;
+		g_autoptr(XbQuery) query = xb_query_new (silo, queries[i].xpath, &error_query);
+		if (query != NULL) {
+			GsAppstreamSearchHelper *helper = g_new0 (GsAppstreamSearchHelper, 1);
+			helper->match_value = queries[i].match_value;
+			helper->query = g_steal_pointer (&query);
+			g_ptr_array_add (array, helper);
+		} else {
+			g_debug ("ignoring: %s", error_query->message);
+		}
+	}
+
+	/* get all components */
+	components = xb_silo_query (silo, "components/component", 0, &error_local);
+	if (components == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < components->len; i++) {
+		XbNode *component = g_ptr_array_index (components, i);
+		guint16 match_value = gs_appstream_silo_search_component (array, component, values);
+		if (match_value != 0) {
+			g_autoptr(GsApp) app = gs_appstream_create_app (plugin, silo, component, error);
+			if (app == NULL)
+				return FALSE;
+			if (gs_app_has_quirk (app, GS_APP_QUIRK_IS_WILDCARD)) {
+				g_debug ("not returning wildcard %s",
+					 gs_app_get_unique_id (app));
+				continue;
+			}
+			g_debug ("add %s", gs_app_get_unique_id (app));
+			gs_app_set_match_value (app, match_value);
+			gs_app_list_add (list, app);
+		}
+	}
+	g_debug ("search took %fms", g_timer_elapsed (timer, NULL) * 1000);
+	return TRUE;
+}
+
+gboolean
+gs_appstream_add_category_apps (GsPlugin *plugin,
+				XbSilo *silo,
+				GsCategory *category,
+				GsAppList *list,
+				GCancellable *cancellable,
+				GError **error)
+{
+	GPtrArray *desktop_groups;
+	g_autoptr(GError) error_local = NULL;
+
+	desktop_groups = gs_category_get_desktop_groups (category);
+	if (desktop_groups->len == 0) {
+		g_warning ("no desktop_groups for %s", gs_category_get_id (category));
+		return TRUE;
+	}
+	for (guint j = 0; j < desktop_groups->len; j++) {
+		const gchar *desktop_group = g_ptr_array_index (desktop_groups, j);
+		g_autofree gchar *xpath = NULL;
+		g_auto(GStrv) split = g_strsplit (desktop_group, "::", -1);
+		g_autoptr(GPtrArray) components = NULL;
+
+		/* generate query */
+		if (g_strv_length (split) == 1) {
+			xpath = g_strdup_printf ("components/component/categories/"
+						 "category[text()='%s']/../..",
+						 split[0]);
+		} else if (g_strv_length (split) == 2) {
+			xpath = g_strdup_printf ("components/component/categories/"
+						 "category[text()='%s']/../"
+						 "category[text()='%s']/../..",
+						 split[0], split[1]);
+		}
+		components = xb_silo_query (silo, xpath, 0, &error_local);
+		if (components == NULL) {
+			if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+				return TRUE;
+			if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+				return TRUE;
+			g_propagate_error (error, g_steal_pointer (&error_local));
+			return FALSE;
+		}
+
+		/* create app */
+		for (guint i = 0; i < components->len; i++) {
+			XbNode *component = g_ptr_array_index (components, i);
+			g_autoptr(GsApp) app = NULL;
+			const gchar *id = xb_node_query_text (component, "id", NULL);
+			if (id == NULL)
+				continue;
+			app = gs_app_new (id);
+			gs_app_add_quirk (app, GS_APP_QUIRK_IS_WILDCARD);
+			gs_app_list_add (list, app);
+		}
+
+	}
+	return TRUE;
+}
+
+static guint
+gs_appstream_count_component_for_groups (GsPlugin *plugin, XbSilo *silo, const gchar *desktop_group)
+{
+	guint limit = 10;
+	g_autofree gchar *xpath = NULL;
+	g_auto(GStrv) split = g_strsplit (desktop_group, "::", -1);
+	g_autoptr(GPtrArray) array = NULL;
+	g_autoptr(GError) error_local = NULL;
+
+	if (g_strv_length (split) == 1) { /* "all" group for a parent category */
+		xpath = g_strdup_printf ("components/component/categories/"
+					 "category[text()='%s']/../..",
+					 split[0]);
+	} else if (g_strv_length (split) == 2) {
+		xpath = g_strdup_printf ("components/component/categories/"
+					 "category[text()='%s']/../"
+					 "category[text()='%s']/../..",
+					 split[0], split[1]);
+	} else {
+		return 0;
+	}
+
+	array = xb_silo_query (silo, xpath, limit, &error_local);
+	if (array == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return 0;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return 0;
+		g_warning ("%s", error_local->message);
+		return 0;
+	}
+	return array->len;
+}
+
+/* we're not actually adding categories here, we're just setting the number of
+ * applications available in each category */
+gboolean
+gs_appstream_add_categories (GsPlugin *plugin,
+			     XbSilo *silo,
+			     GPtrArray *list,
+			     GCancellable *cancellable,
+			     GError **error)
+{
+	for (guint j = 0; j < list->len; j++) {
+		GsCategory *parent = GS_CATEGORY (g_ptr_array_index (list, j));
+		GPtrArray *children = gs_category_get_children (parent);
+
+		for (guint i = 0; i < children->len; i++) {
+			GsCategory *cat = g_ptr_array_index (children, i);
+			GPtrArray *groups = gs_category_get_desktop_groups (cat);
+			for (guint k = 0; k < groups->len; k++) {
+				const gchar *group = g_ptr_array_index (groups, k);
+				guint cnt = gs_appstream_count_component_for_groups (plugin, silo, group);
+				for (guint l = 0; l < cnt; l++) {
+					gs_category_increment_size (parent);
+					if (children->len > 1) {
+						/* Parent category has multiple groups, so increment
+						 * each group's size too */
+						gs_category_increment_size (cat);
+					}
+				}
+			}
+		}
+		continue;
+	}
+	return TRUE;
+}
+
+gboolean
+gs_appstream_add_popular (GsPlugin *plugin,
+			  XbSilo *silo,
+			  GsAppList *list,
+			  GCancellable *cancellable,
+			  GError **error)
+{
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) array = NULL;
+
+	/* find out how many packages are in each category */
+	array = xb_silo_query (silo,
+			       "components/component/kudos/"
+			       "kudo[text()='GnomeSoftware::popular']/../..",
+			       0, &error_local);
+	if (array == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < array->len; i++) {
+		g_autoptr(GsApp) app = NULL;
+		XbNode *component = g_ptr_array_index (array, i);
+		const gchar *component_id = xb_node_query_text (component, "id", NULL);
+		if (component_id == NULL)
+			continue;
+		app = gs_app_new (component_id);
+		gs_app_add_quirk (app, GS_APP_QUIRK_IS_WILDCARD);
+		gs_app_list_add (list, app);
+	}
+	return TRUE;
+}
+
+gboolean
+gs_appstream_add_recent (GsPlugin *plugin,
+			 XbSilo *silo,
+			 GsAppList *list,
+			 guint64 age,
+			 GCancellable *cancellable,
+			 GError **error)
+{
+	guint64 now = (guint64) g_get_real_time () / G_USEC_PER_SEC;
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) array = NULL;
+
+	/* use predicate conditions to the max */
+	xpath = g_strdup_printf ("components/component/releases/"
+				 "release[@timestamp>%" G_GUINT64_FORMAT "]/../..",
+				 now - (30 * 24 * 60 * 60));
+	array = xb_silo_query (silo, xpath, 0, &error_local);
+	if (array == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < array->len; i++) {
+		XbNode *component = g_ptr_array_index (array, i);
+		g_autoptr(GsApp) app = gs_appstream_create_app (plugin, silo, component, error);
+		if (app == NULL)
+			return FALSE;
+		gs_app_list_add (list, app);
+	}
+	return TRUE;
+}
+
+gboolean
+gs_appstream_add_alternates (GsPlugin *plugin,
+			     XbSilo *silo,
+			     GsApp *app,
+			     GsAppList *list,
+			     GCancellable *cancellable,
+			     GError **error)
+{
+	GPtrArray *sources = gs_app_get_sources (app);
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) ids = NULL;
+	g_autoptr(GString) xpath = g_string_new (NULL);
+
+	/* probably a package we know nothing about */
+	if (gs_app_get_id (app) == NULL)
+		return TRUE;
+
+	/* actual ID */
+	xb_string_append_union (xpath, "components/component/id[text()='%s']",
+				gs_app_get_id (app));
+
+	/* new ID -> old ID */
+	xb_string_append_union (xpath, "components/component/id[text()='%s']/../provides/id",
+				gs_app_get_id (app));
+
+	/* old ID -> new ID */
+	xb_string_append_union (xpath, "components/component/provides/id[text()='%s']/../../id",
+				gs_app_get_id (app));
+
+	/* find apps that use the same pkgname */
+	for (guint j = 0; j < sources->len; j++) {
+		const gchar *source = g_ptr_array_index (sources, j);
+		g_autofree gchar *source_safe = xb_string_escape (source);
+		xb_string_append_union (xpath,
+					"components/component/pkgname[text()='%s']/../id",
+					source_safe);
+	}
+
+	/* do a big query, and return all the unique results */
+	ids = xb_silo_query (silo, xpath->str, 0, &error_local);
+	if (ids == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < ids->len; i++) {
+		XbNode *n = g_ptr_array_index (ids, i);
+		g_autoptr(GsApp) app2 = NULL;
+		app2 = gs_app_new (xb_node_get_text (n));
+		gs_app_add_quirk (app2, GS_APP_QUIRK_IS_WILDCARD);
+		gs_app_list_add (list, app2);
+	}
+	return TRUE;
+}
+
+gboolean
+gs_appstream_add_featured (GsPlugin *plugin,
+			   XbSilo *silo,
+			   GsAppList *list,
+			   GCancellable *cancellable,
+			   GError **error)
+{
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GPtrArray) array = NULL;
+
+	/* find out how many packages are in each category */
+	array = xb_silo_query (silo,
+			       "components/component/custom/"
+			       "value[@key='GnomeSoftware::FeatureTile-css']/../..",
+			       0, &error_local);
+	if (array == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < array->len; i++) {
+		g_autoptr(GsApp) app = NULL;
+		XbNode *component = g_ptr_array_index (array, i);
+		const gchar *component_id = xb_node_query_text (component, "id", NULL);
+		if (component_id == NULL)
+			continue;
+		app = gs_app_new (component_id);
+		gs_app_add_quirk (app, GS_APP_QUIRK_IS_WILDCARD);
+		if (!gs_appstream_copy_metadata (app, component, error))
+			return FALSE;
+		gs_app_list_add (list, app);
+	}
+	return TRUE;
+}
+
+void
+gs_appstream_component_add_keyword (XbBuilderNode *component, const gchar *str)
+{
+	g_autoptr(XbBuilderNode) keyword = NULL;
+	g_autoptr(XbBuilderNode) keywords = NULL;
+
+	/* create <keywords> if it does not already exist */
+	keywords = xb_builder_node_get_child (component, "keywords", NULL);
+	if (keywords == NULL)
+		keywords = xb_builder_node_insert (component, "keywords", NULL);
+
+	/* create <keyword>str</keyword> if it does not already exist */
+	keyword = xb_builder_node_get_child (keywords, "keyword", str);
+	if (keyword == NULL) {
+		keyword = xb_builder_node_insert (keywords, "keyword", NULL);
+		xb_builder_node_set_text (keyword, str, -1);
+	}
+}
+
+void
+gs_appstream_component_add_provide (XbBuilderNode *component, const gchar *str)
+{
+	g_autoptr(XbBuilderNode) provide = NULL;
+	g_autoptr(XbBuilderNode) provides = NULL;
+
+	/* create <provides> if it does not already exist */
+	provides = xb_builder_node_get_child (component, "provides", NULL);
+	if (provides == NULL)
+		provides = xb_builder_node_insert (component, "provides", NULL);
+
+	/* create <id>str</id> if it does not already exist */
+	provide = xb_builder_node_get_child (provides, "id", str);
+	if (provide == NULL) {
+		provide = xb_builder_node_insert (provides, "id", NULL);
+		xb_builder_node_set_text (provide, str, -1);
+	}
+}
+
+void
+gs_appstream_component_add_category (XbBuilderNode *component, const gchar *str)
+{
+	g_autoptr(XbBuilderNode) category = NULL;
+	g_autoptr(XbBuilderNode) categories = NULL;
+
+	/* create <categories> if it does not already exist */
+	categories = xb_builder_node_get_child (component, "categories", NULL);
+	if (categories == NULL)
+		categories = xb_builder_node_insert (component, "categories", NULL);
+
+	/* create <category>str</category> if it does not already exist */
+	category = xb_builder_node_get_child (categories, "category", str);
+	if (category == NULL) {
+		category = xb_builder_node_insert (categories, "category", NULL);
+		xb_builder_node_set_text (category, str, -1);
+	}
+}
+
+void
+gs_appstream_component_add_icon (XbBuilderNode *component, const gchar *str)
+{
+	g_autoptr(XbBuilderNode) icon = NULL;
+
+	/* create <icon>str</icon> if it does not already exist */
+	icon = xb_builder_node_get_child (component, "icon", NULL);
+	if (icon == NULL) {
+		icon = xb_builder_node_insert (component, "icon",
+					       "type", "stock",
+					       NULL);
+		xb_builder_node_set_text (icon, str, -1);
+	}
+}
+
+void
+gs_appstream_component_add_extra_info (GsPlugin *plugin, XbBuilderNode *component)
+{
+	const gchar *kind = xb_builder_node_get_attr (component, "type");
+
+	/* add the gnome-software-specific 'Addon' group and ensure they
+	 * all have an icon set */
+	switch (as_app_kind_from_string (kind)) {
+	case AS_APP_KIND_WEB_APP:
+		gs_appstream_component_add_keyword (component, kind);
+		break;
+	case AS_APP_KIND_FONT:
+		gs_appstream_component_add_category (component, "Addon");
+		gs_appstream_component_add_category (component, "Font");
+		break;
+	case AS_APP_KIND_DRIVER:
+		gs_appstream_component_add_category (component, "Addon");
+		gs_appstream_component_add_category (component, "Driver");
+		gs_appstream_component_add_icon (component, "application-x-firmware-symbolic");
+		break;
+	case AS_APP_KIND_LOCALIZATION:
+		gs_appstream_component_add_category (component, "Addon");
+		gs_appstream_component_add_category (component, "Localization");
+		gs_appstream_component_add_icon (component, "accessories-dictionary-symbolic");
+		break;
+	case AS_APP_KIND_CODEC:
+		gs_appstream_component_add_category (component, "Addon");
+		gs_appstream_component_add_category (component, "Codec");
+		gs_appstream_component_add_icon (component, "application-x-addon");
+		break;
+	case AS_APP_KIND_INPUT_METHOD:
+		gs_appstream_component_add_keyword (component, kind);
+		gs_appstream_component_add_category (component, "Addon");
+		gs_appstream_component_add_category (component, "InputSource");
+		gs_appstream_component_add_icon (component, "system-run-symbolic");
+		break;
+	case AS_APP_KIND_FIRMWARE:
+		gs_appstream_component_add_icon (component, "system-run-symbolic");
+		break;
+	default:
+		break;
+	}
+}
diff --git a/plugins/core/gs-appstream.h b/plugins/core/gs-appstream.h
new file mode 100644
index 0000000..d6e9a0b
--- /dev/null
+++ b/plugins/core/gs-appstream.h
@@ -0,0 +1,76 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2015-2017 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#pragma once
+
+#include <gnome-software.h>
+#include <xmlb.h>
+
+G_BEGIN_DECLS
+
+GsApp		*gs_appstream_create_app		(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 XbNode		*component,
+							 GError		**error);
+gboolean	 gs_appstream_refine_app		(GsPlugin	*plugin,
+							 GsApp		*app,
+							 XbSilo		*silo,
+							 XbNode		*component,
+							 GsPluginRefineFlags flags,
+							 GError		**error);
+gboolean	 gs_appstream_search			(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 const gchar * const *values,
+							 GsAppList	*list,
+							 GCancellable	*cancellable,
+							 GError		**error);
+gboolean	 gs_appstream_add_categories		(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 GPtrArray	*list,
+							 GCancellable	*cancellable,
+							 GError		**error);
+gboolean	 gs_appstream_add_category_apps		(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 GsCategory	*category,
+							 GsAppList	*list,
+							 GCancellable	*cancellable,
+							 GError		**error);
+gboolean	 gs_appstream_add_popular		(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 GsAppList	*list,
+							 GCancellable	*cancellable,
+							 GError		**error);
+gboolean	 gs_appstream_add_featured		(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 GsAppList	*list,
+							 GCancellable	*cancellable,
+							 GError		**error);
+gboolean	 gs_appstream_add_alternates		(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 GsApp		*app,
+							 GsAppList	*list,
+							 GCancellable	*cancellable,
+							 GError		**error);
+gboolean	 gs_appstream_add_recent		(GsPlugin	*plugin,
+							 XbSilo		*silo,
+							 GsAppList	*list,
+							 guint64	 age,
+							 GCancellable	*cancellable,
+							 GError		**error);
+void		 gs_appstream_component_add_extra_info	(GsPlugin	*plugin,
+							 XbBuilderNode	*component);
+void		 gs_appstream_component_add_keyword	(XbBuilderNode	*component,
+							 const gchar	*str);
+void		 gs_appstream_component_add_category	(XbBuilderNode	*component,
+							 const gchar	*str);
+void		 gs_appstream_component_add_icon	(XbBuilderNode	*component,
+							 const gchar	*str);
+void		 gs_appstream_component_add_provide	(XbBuilderNode	*component,
+							 const gchar	*str);
+
+G_END_DECLS
diff --git a/plugins/core/gs-desktop-common.c b/plugins/core/gs-desktop-common.c
new file mode 100644
index 0000000..27e11bd
--- /dev/null
+++ b/plugins/core/gs-desktop-common.c
@@ -0,0 +1,333 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2015-2016 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include "config.h"
+
+#include <glib/gi18n.h>
+
+#include "gs-desktop-common.h"
+
+/* AudioVideo */
+static const GsDesktopMap map_audiovisual[] = {
+	{ "all",		NC_("Menu of Audio & Video", "All"),
+					{ "AudioVideo",
+					  NULL } },
+	{ "featured",		NC_("Menu of Audio & Video", "Featured"),
+					{ "AudioVideo::Featured",
+					  NULL} },
+	{ "creation-editing",	NC_("Menu of Audio & Video", "Audio Creation & Editing"),
+					{ "AudioVideo::AudioVideoEditing",
+					  "AudioVideo::Midi",
+					  "AudioVideo::DiscBurning",
+					  "AudioVideo::Sequencer",
+					  NULL} },
+	{ "music-players",	NC_("Menu of Audio & Video", "Music Players"),
+					{ "AudioVideo::Music",
+					  "AudioVideo::Player",
+					  NULL} },
+	{ NULL }
+};
+
+/* Development */
+static const GsDesktopMap map_developertools[] = {
+	{ "all",		NC_("Menu of Developer Tools", "All"),
+					{ "Development",
+					  NULL } },
+	{ "featured",		NC_("Menu of Developer Tools", "Featured"),
+					{ "Development::Featured",
+					  NULL} },
+	{ "debuggers",		NC_("Menu of Developer Tools", "Debuggers"),
+					{ "Development::Debugger",
+					  NULL} },
+	{ "ide",		NC_("Menu of Developer Tools", "IDEs"),
+					{ "Development::IDE",
+					  "Development::GUIDesigner",
+					  NULL} },
+	{ NULL }
+};
+
+/* Education & Science */
+static const GsDesktopMap map_education_science[] = {
+	{ "all",		NC_("Menu of Education & Science", "All"),
+					{ "Education",
+					  "Science",
+					  NULL } },
+	{ "featured",		NC_("Menu of Education & Science", "Featured"),
+					{ "Education::Featured",
+					  "Science::Featured",
+					  NULL} },
+	{ "artificial-intelligence", NC_("Menu of Education & Science", "Artificial Intelligence"),
+					{ "Science::ArtificialIntelligence",
+					  NULL} },
+	{ "astronomy",		NC_("Menu of Education & Science", "Astronomy"),
+					{ "Education::Astronomy",
+					  "Science::Astronomy",
+					  NULL} },
+	{ "chemistry",		NC_("Menu of Education & Science", "Chemistry"),
+					{ "Education::Chemistry",
+					  "Science::Chemistry",
+					  NULL} },
+	{ "languages",		NC_("Menu of Education & Science", "Languages"),
+					{ "Education::Languages",
+					  "Education::Literature",
+					  NULL} },
+	{ "math",		NC_("Menu of Education & Science", "Math"),
+					{ "Education::Math",
+					  "Education::NumericalAnalysis",
+					  "Science::Math",
+					  "Science::Physics",
+					  "Science::NumericalAnalysis",
+					  NULL} },
+	{ "robotics",		NC_("Menu of Education & Science", "Robotics"),
+					{ "Science::Robotics",
+					  NULL} },
+
+	{ NULL }
+};
+
+/* Games */
+static const GsDesktopMap map_games[] = {
+	{ "all",		NC_("Menu of Games", "All"),
+					{ "Game",
+					  NULL } },
+	{ "featured",		NC_("Menu of Games", "Featured"),
+					{ "Game::Featured",
+					  NULL} },
+	{ "action",		NC_("Menu of Games", "Action"),
+					{ "Game::ActionGame",
+					  NULL} },
+	{ "adventure",		NC_("Menu of Games", "Adventure"),
+					{ "Game::AdventureGame",
+					  NULL} },
+	{ "arcade",		NC_("Menu of Games", "Arcade"),
+					{ "Game::ArcadeGame",
+					  NULL} },
+	{ "blocks",		NC_("Menu of Games", "Blocks"),
+					{ "Game::BlocksGame",
+					  NULL} },
+	{ "board",		NC_("Menu of Games", "Board"),
+					{ "Game::BoardGame",
+					  NULL} },
+	{ "card",		NC_("Menu of Games", "Card"),
+					{ "Game::CardGame",
+					  NULL} },
+	{ "emulator",		NC_("Menu of Games", "Emulators"),
+					{ "Game::Emulator",
+					  NULL} },
+	{ "kids",		NC_("Menu of Games", "Kids"),
+					{ "Game::KidsGame",
+					  NULL} },
+	{ "logic",		NC_("Menu of Games", "Logic"),
+					{ "Game::LogicGame",
+					  NULL} },
+	{ "role-playing",	NC_("Menu of Games", "Role Playing"),
+					{ "Game::RolePlaying",
+					  NULL} },
+	{ "sports",		NC_("Menu of Games", "Sports"),
+					{ "Game::SportsGame",
+					  "Game::Simulation",
+					  NULL} },
+	{ "strategy",		NC_("Menu of Games", "Strategy"),
+					{ "Game::StrategyGame",
+					  NULL} },
+	{ NULL }
+};
+
+/* Graphics */
+static const GsDesktopMap map_graphics[] = {
+	{ "all",		NC_("Menu of Graphics & Photography", "All"),
+					{ "Graphics",
+					  NULL } },
+	{ "featured",		NC_("Menu of Graphics & Photography", "Featured"),
+					{ "Graphics::Featured",
+					  NULL} },
+	{ "3d",			NC_("Menu of Graphics & Photography", "3D Graphics"),
+					{ "Graphics::3DGraphics",
+					  NULL} },
+	{ "photography",	NC_("Menu of Graphics & Photography", "Photography"),
+					{ "Graphics::Photography",
+					  NULL} },
+	{ "scanning",		NC_("Menu of Graphics & Photography", "Scanning"),
+					{ "Graphics::Scanning",
+					  NULL} },
+	{ "vector",		NC_("Menu of Graphics & Photography", "Vector Graphics"),
+					{ "Graphics::VectorGraphics",
+					  NULL} },
+	{ "viewers",		NC_("Menu of Graphics & Photography", "Viewers"),
+					{ "Graphics::Viewer",
+					  NULL} },
+	{ NULL }
+};
+
+/* Office */
+static const GsDesktopMap map_productivity[] = {
+	{ "all",		NC_("Menu of Productivity", "All"),
+					{ "Office",
+					  NULL } },
+	{ "featured",		NC_("Menu of Productivity", "Featured"),
+					{ "Office::Featured",
+					  NULL} },
+	{ "calendar",		NC_("Menu of Productivity", "Calendar"),
+					{ "Office::Calendar",
+					  "Office::ProjectManagement",
+					  NULL} },
+	{ "database",		NC_("Menu of Productivity", "Database"),
+					{ "Office::Database",
+					  NULL} },
+	{ "finance",		NC_("Menu of Productivity", "Finance"),
+					{ "Office::Finance",
+					  "Office::Spreadsheet",
+					  NULL} },
+	{ "word-processor",	NC_("Menu of Productivity", "Word Processor"),
+					{ "Office::WordProcessor",
+					  "Office::Dictionary",
+					  NULL} },
+	{ NULL }
+};
+
+/* Addons */
+static const GsDesktopMap map_addons[] = {
+	{ "fonts",		NC_("Menu of Add-ons", "Fonts"),
+					{ "Addon::Font",
+					  NULL} },
+	{ "codecs",		NC_("Menu of Add-ons", "Codecs"),
+					{ "Addon::Codec",
+					  NULL} },
+	{ "input-sources",	NC_("Menu of Add-ons", "Input Sources"),
+					{ "Addon::InputSource",
+					  NULL} },
+	{ "language-packs",	NC_("Menu of Add-ons", "Language Packs"),
+					{ "Addon::LanguagePack",
+					  NULL} },
+	{ "localization",	NC_("Menu of Add-ons", "Localization"),
+					{ "Addon::Localization",
+					  NULL} },
+	{ "drivers",		NC_("Menu of Add-ons", "Hardware Drivers"),
+					{ "Addon::Driver",
+					  NULL} },
+	{ NULL }
+};
+
+/* Communication */
+static const GsDesktopMap map_communication[] = {
+	{ "all",		NC_("Menu of Communication & News", "All"),
+					{ "Network",
+					  NULL } },
+	{ "featured",		NC_("Menu of Communication & News", "Featured"),
+					{ "Network::Featured",
+					  NULL} },
+	{ "chat",		NC_("Menu of Communication & News", "Chat"),
+					{ "Network::Chat",
+					  "Network::IRCClient",
+					  "Network::Telephony",
+					  "Network::VideoConference",
+					  "Network::Email",
+					  NULL} },
+	{ "news",		NC_("Menu of Communication & News", "News"),
+					{ "Network::Feed",
+					  "Network::News",
+					  NULL} },
+	{ "web-browsers",	NC_("Menu of Communication & News", "Web Browsers"),
+					{ "Network::WebBrowser",
+					  NULL} },
+	{ NULL }
+};
+
+/* Utility */
+static const GsDesktopMap map_utilities[] = {
+	{ "all",		NC_("Menu of Utilities", "All"),
+					{ "Utility",
+					  NULL } },
+	{ "featured",		NC_("Menu of Utilities", "Featured"),
+					{ "Utility::Featured",
+					  NULL} },
+	{ "text-editors",	NC_("Menu of Utilities", "Text Editors"),
+					{ "Utility::TextEditor",
+					  NULL} },
+	{ NULL }
+};
+
+/* Reference */
+static const GsDesktopMap map_reference[] = {
+	{ "all",		NC_("Menu of Reference", "All"),
+					{ "Reference",
+					  NULL } },
+	{ "featured",		NC_("Menu of Reference", "Featured"),
+					{ "Reference::Featured",
+					  NULL} },
+	{ "art",		NC_("Menu of Art", "Art"),
+					{ "Reference::Art",
+					  NULL} },
+	{ "biography",		NC_("Menu of Reference", "Biography"),
+					{ "Reference::Biography",
+					  NULL} },
+	{ "comics",		NC_("Menu of Reference", "Comics"),
+					{ "Reference::Comics",
+					  NULL} },
+	{ "fiction",		NC_("Menu of Reference", "Fiction"),
+					{ "Reference::Fiction",
+					  NULL} },
+	{ "health",		NC_("Menu of Reference", "Health"),
+					{ "Reference::Health",
+					  NULL} },
+	{ "history",		NC_("Menu of Reference", "History"),
+					{ "Reference::History",
+					  NULL} },
+	{ "lifestyle",		NC_("Menu of Reference", "Lifestyle"),
+					{ "Reference::Lifestyle",
+					  NULL} },
+	{ "politics",		NC_("Menu of Reference", "Politics"),
+					{ "Reference::Politics",
+					  NULL} },
+	{ "sports",		NC_("Menu of Reference", "Sports"),
+					{ "Reference::Sports",
+					  NULL} },
+	{ NULL }
+};
+
+/* main categories */
+/* Please keep category name and subcategory context synchronized!!! */
+static const GsDesktopData msdata[] = {
+	/* TRANSLATORS: this is the menu spec main category for Audio & Video */
+	{ "audio-video",	map_audiovisual,	N_("Audio & Video"),
+				"folder-music-symbolic", 100 },
+	/* TRANSLATORS: this is the menu spec main category for Development */
+	{ "developer-tools",	map_developertools,	N_("Developer Tools"),
+				"applications-engineering-symbolic", 40 },
+	/* TRANSLATORS: this is the menu spec main category for Education & Science */
+	{ "education-science",		map_education_science,	N_("Education & Science"),
+				"system-help-symbolic", 30 },
+	/* TRANSLATORS: this is the menu spec main category for Game */
+	{ "games",		map_games,		N_("Games"),
+				"applications-games-symbolic", 70 },
+	/* TRANSLATORS: this is the menu spec main category for Graphics */
+	{ "graphics",		map_graphics,		N_("Graphics & Photography"),
+				"applications-graphics-symbolic", 60 },
+	/* TRANSLATORS: this is the menu spec main category for Office */
+	{ "productivity",	map_productivity,	N_("Productivity"),
+				"text-editor-symbolic", 80 },
+	/* TRANSLATORS: this is the menu spec main category for Add-ons */
+	{ "addons",		map_addons,		N_("Add-ons"),
+				"application-x-addon-symbolic", 50 },
+	/* TRANSLATORS: this is the menu spec main category for Communication */
+	{ "communication",	map_communication,	N_("Communication & News"),
+				"user-available-symbolic", 90 },
+	/* TRANSLATORS: this is the menu spec main category for Reference */
+	{ "reference",		map_reference,		N_("Reference"),
+				"view-dual-symbolic", 0 },
+	/* TRANSLATORS: this is the menu spec main category for Utilities */
+	{ "utilities",		map_utilities,		N_("Utilities"),
+				"applications-utilities-symbolic", 10 },
+	{ NULL }
+};
+
+const GsDesktopData *
+gs_desktop_get_data (void)
+{
+	return msdata;
+}
diff --git a/plugins/core/gs-desktop-common.h b/plugins/core/gs-desktop-common.h
new file mode 100644
index 0000000..3f5d378
--- /dev/null
+++ b/plugins/core/gs-desktop-common.h
@@ -0,0 +1,31 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2011-2016 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+typedef struct {
+	const gchar	*id;
+	const gchar	*name;
+	const gchar	*fdo_cats[16];
+} GsDesktopMap;
+
+typedef struct {
+	const gchar	*id;
+	const GsDesktopMap *mapping;
+	const gchar	*name;
+	const gchar	*icon;
+	gint		 score;
+} GsDesktopData;
+
+const GsDesktopData	*gs_desktop_get_data		(void);
+
+G_END_DECLS
diff --git a/plugins/core/gs-plugin-appstream.c b/plugins/core/gs-plugin-appstream.c
new file mode 100644
index 0000000..6a28577
--- /dev/null
+++ b/plugins/core/gs-plugin-appstream.c
@@ -0,0 +1,1092 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2013-2014 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2015-2019 Kalev Lember <klember@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <glib/gi18n.h>
+#include <gnome-software.h>
+#include <xmlb.h>
+
+#include "gs-appstream.h"
+
+/*
+ * SECTION:
+ * Uses offline AppStream data to populate and refine package results.
+ *
+ * This plugin calls UpdatesChanged() if any of the AppStream stores are
+ * changed in any way.
+ *
+ * Methods:     | AddCategory
+ * Refines:     | [source]->[name,summary,pixbuf,id,kind]
+ */
+
+struct GsPluginData {
+	XbSilo			*silo;
+	GRWLock			 silo_lock;
+	GSettings		*settings;
+};
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_alloc_data (plugin, sizeof(GsPluginData));
+
+	/* XbSilo needs external locking as we destroy the silo and build a new
+	 * one when something changes */
+	g_rw_lock_init (&priv->silo_lock);
+
+	/* need package name */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "dpkg");
+
+	/* require settings */
+	priv->settings = g_settings_new ("org.gnome.software");
+}
+
+void
+gs_plugin_destroy (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_object_unref (priv->silo);
+	g_object_unref (priv->settings);
+	g_rw_lock_clear (&priv->silo_lock);
+}
+
+static const gchar *
+gs_plugin_appstream_convert_component_kind (const gchar *kind)
+{
+	if (g_strcmp0 (kind, "web-application") == 0)
+		return "webapp";
+	if (g_strcmp0 (kind, "console-application") == 0)
+		return "console";
+	return kind;
+}
+
+static gboolean
+gs_plugin_appstream_upgrade_cb (XbBuilderFixup *self,
+				XbBuilderNode *bn,
+				gpointer user_data,
+				GError **error)
+{
+	if (g_strcmp0 (xb_builder_node_get_element (bn), "application") == 0) {
+		g_autoptr(XbBuilderNode) id = xb_builder_node_get_child (bn, "id", NULL);
+		g_autofree gchar *kind = NULL;
+		if (id != NULL) {
+			kind = g_strdup (xb_builder_node_get_attr (id, "type"));
+			xb_builder_node_remove_attr (id, "type");
+		}
+		if (kind != NULL)
+			xb_builder_node_set_attr (bn, "type", kind);
+		xb_builder_node_set_element (bn, "component");
+	} else if (g_strcmp0 (xb_builder_node_get_element (bn), "metadata") == 0) {
+		xb_builder_node_set_element (bn, "custom");
+	} else if (g_strcmp0 (xb_builder_node_get_element (bn), "component") == 0) {
+		const gchar *type_old = xb_builder_node_get_attr (bn, "type");
+		const gchar *type_new = gs_plugin_appstream_convert_component_kind (type_old);
+		if (type_old != type_new)
+			xb_builder_node_set_attr (bn, "type", type_new);
+	}
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_appstream_add_icons_cb (XbBuilderFixup *self,
+				  XbBuilderNode *bn,
+				  gpointer user_data,
+				  GError **error)
+{
+	GsPlugin *plugin = GS_PLUGIN (user_data);
+	if (g_strcmp0 (xb_builder_node_get_element (bn), "component") != 0)
+		return TRUE;
+	gs_appstream_component_add_extra_info (plugin, bn);
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_appstream_add_origin_keyword_cb (XbBuilderFixup *self,
+					   XbBuilderNode *bn,
+					   gpointer user_data,
+					   GError **error)
+{
+	if (g_strcmp0 (xb_builder_node_get_element (bn), "components") == 0) {
+		const gchar *origin = xb_builder_node_get_attr (bn, "origin");
+		GPtrArray *components = xb_builder_node_get_children (bn);
+		if (origin == NULL || origin[0] == '\0')
+			return TRUE;
+		g_debug ("origin %s has %u components", origin, components->len);
+		if (components->len < 200) {
+			for (guint i = 0; i < components->len; i++) {
+				XbBuilderNode *component = g_ptr_array_index (components, i);
+				gs_appstream_component_add_keyword (component, origin);
+			}
+		}
+	}
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_appstream_load_appdata_fn (GsPlugin *plugin,
+				     XbBuilder *builder,
+				     const gchar *filename,
+				     GCancellable *cancellable,
+				     GError **error)
+{
+	g_autoptr(GFile) file = g_file_new_for_path (filename);
+	g_autoptr(XbBuilderFixup) fixup = NULL;
+	g_autoptr(XbBuilderNode) info = NULL;
+	g_autoptr(XbBuilderSource) source = xb_builder_source_new ();
+
+	/* add source */
+	if (!xb_builder_source_load_file (source, file,
+#if LIBXMLB_CHECK_VERSION(0, 2, 0)
+					  XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY,
+#else
+					  XB_BUILDER_SOURCE_FLAG_WATCH_FILE,
+#endif
+					  cancellable,
+					  error)) {
+		return FALSE;
+	}
+
+	/* fix up any legacy installed files */
+	fixup = xb_builder_fixup_new ("AppStreamUpgrade2",
+				      gs_plugin_appstream_upgrade_cb,
+				      plugin, NULL);
+	xb_builder_fixup_set_max_depth (fixup, 3);
+	xb_builder_source_add_fixup (source, fixup);
+
+	/* add metadata */
+	info = xb_builder_node_insert (NULL, "info", NULL);
+	xb_builder_node_insert_text (info, "filename", filename, NULL);
+	xb_builder_source_set_info (source, info);
+
+	/* success */
+	xb_builder_import_source (builder, source);
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_appstream_load_appdata (GsPlugin *plugin,
+				  XbBuilder *builder,
+				  const gchar *path,
+				  GCancellable *cancellable,
+				  GError **error)
+{
+	const gchar *fn;
+	g_autoptr(GDir) dir = NULL;
+	g_autoptr(GFile) parent = g_file_new_for_path (path);
+	if (!g_file_query_exists (parent, cancellable))
+		return TRUE;
+
+	dir = g_dir_open (path, 0, error);
+	if (dir == NULL)
+		return FALSE;
+
+	while ((fn = g_dir_read_name (dir)) != NULL) {
+		if (g_str_has_suffix (fn, ".appdata.xml") ||
+		    g_str_has_suffix (fn, ".metainfo.xml")) {
+			g_autofree gchar *filename = g_build_filename (path, fn, NULL);
+			g_autoptr(GError) error_local = NULL;
+			if (!gs_plugin_appstream_load_appdata_fn (plugin,
+								  builder,
+								  filename,
+								  cancellable,
+								  &error_local)) {
+				g_debug ("ignoring %s: %s", filename, error_local->message);
+				continue;
+			}
+		}
+	}
+
+	/* success */
+	return TRUE;
+}
+
+static GInputStream *
+gs_plugin_appstream_load_desktop_cb (XbBuilderSource *self,
+				     XbBuilderSourceCtx *ctx,
+				     gpointer user_data,
+				     GCancellable *cancellable,
+				     GError **error)
+{
+	GString *xml;
+	g_autoptr(AsApp) app = as_app_new ();
+	g_autoptr(GBytes) bytes = NULL;
+	bytes = xb_builder_source_ctx_get_bytes (ctx, cancellable, error);
+	if (bytes == NULL)
+		return NULL;
+	as_app_set_id (app, xb_builder_source_ctx_get_filename (ctx));
+	if (!as_app_parse_data (app, bytes, AS_APP_PARSE_FLAG_USE_FALLBACKS, error))
+		return NULL;
+	xml = as_app_to_xml (app, error);
+	if (xml == NULL)
+		return NULL;
+	g_string_prepend (xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+	return g_memory_input_stream_new_from_data (g_string_free (xml, FALSE), -1, g_free);
+}
+
+static gboolean
+gs_plugin_appstream_load_desktop_fn (GsPlugin *plugin,
+				     XbBuilder *builder,
+				     const gchar *filename,
+				     GCancellable *cancellable,
+				     GError **error)
+{
+	g_autoptr(GFile) file = g_file_new_for_path (filename);
+	g_autoptr(XbBuilderFixup) fixup = NULL;
+	g_autoptr(XbBuilderNode) info = NULL;
+	g_autoptr(XbBuilderSource) source = xb_builder_source_new ();
+
+	/* add support for desktop files */
+	xb_builder_source_add_adapter (source, "application/x-desktop",
+				       gs_plugin_appstream_load_desktop_cb, NULL, NULL);
+
+	/* add source */
+	if (!xb_builder_source_load_file (source, file,
+#if LIBXMLB_CHECK_VERSION(0, 2, 0)
+					  XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY,
+#else
+					  XB_BUILDER_SOURCE_FLAG_WATCH_FILE,
+#endif
+					  cancellable,
+					  error)) {
+		return FALSE;
+	}
+
+	/* add metadata */
+	info = xb_builder_node_insert (NULL, "info", NULL);
+	xb_builder_node_insert_text (info, "filename", filename, NULL);
+	xb_builder_source_set_info (source, info);
+
+	/* success */
+	xb_builder_import_source (builder, source);
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_appstream_load_desktop (GsPlugin *plugin,
+				  XbBuilder *builder,
+				  const gchar *path,
+				  GCancellable *cancellable,
+				  GError **error)
+{
+	const gchar *fn;
+	g_autoptr(GDir) dir = NULL;
+	g_autoptr(GFile) parent = g_file_new_for_path (path);
+	if (!g_file_query_exists (parent, cancellable))
+		return TRUE;
+
+	dir = g_dir_open (path, 0, error);
+	if (dir == NULL)
+		return FALSE;
+
+	while ((fn = g_dir_read_name (dir)) != NULL) {
+		if (g_str_has_suffix (fn, ".desktop")) {
+			g_autofree gchar *filename = g_build_filename (path, fn, NULL);
+			g_autoptr(GError) error_local = NULL;
+			if (g_strcmp0 (fn, "mimeinfo.cache") == 0)
+				continue;
+			if (!gs_plugin_appstream_load_desktop_fn (plugin,
+								  builder,
+								  filename,
+								  cancellable,
+								  &error_local)) {
+				g_debug ("ignoring %s: %s", filename, error_local->message);
+				continue;
+			}
+		}
+	}
+
+	/* success */
+	return TRUE;
+}
+
+static GInputStream *
+gs_plugin_appstream_load_dep11_cb (XbBuilderSource *self,
+				   XbBuilderSourceCtx *ctx,
+				   gpointer user_data,
+				   GCancellable *cancellable,
+				   GError **error)
+{
+	GString *xml;
+	g_autoptr(AsStore) store = as_store_new ();
+	g_autoptr(GBytes) bytes = NULL;
+	bytes = xb_builder_source_ctx_get_bytes (ctx, cancellable, error);
+	if (bytes == NULL)
+		return NULL;
+	if (!as_store_from_bytes (store, bytes, cancellable, error))
+		return FALSE;
+	xml = as_store_to_xml (store, AS_NODE_INSERT_FLAG_NONE);
+	if (xml == NULL)
+		return NULL;
+	g_string_prepend (xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+	return g_memory_input_stream_new_from_data (g_string_free (xml, FALSE), -1, g_free);
+}
+
+static gboolean
+gs_plugin_appstream_load_appstream_fn (GsPlugin *plugin,
+				       XbBuilder *builder,
+				       const gchar *filename,
+				       GCancellable *cancellable,
+				       GError **error)
+{
+	g_autoptr(GFile) file = g_file_new_for_path (filename);
+	g_autoptr(XbBuilderNode) info = NULL;
+	g_autoptr(XbBuilderFixup) fixup1 = NULL;
+	g_autoptr(XbBuilderFixup) fixup2 = NULL;
+	g_autoptr(XbBuilderFixup) fixup3 = NULL;
+	g_autoptr(XbBuilderSource) source = xb_builder_source_new ();
+
+	/* add support for DEP-11 files */
+	xb_builder_source_add_adapter (source,
+				       "application/x-yaml",
+				       gs_plugin_appstream_load_dep11_cb,
+				       NULL, NULL);
+
+	/* add source */
+	if (!xb_builder_source_load_file (source, file,
+#if LIBXMLB_CHECK_VERSION(0, 2, 0)
+					  XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY,
+#else
+					  XB_BUILDER_SOURCE_FLAG_WATCH_FILE,
+#endif
+					  cancellable,
+					  error)) {
+		return FALSE;
+	}
+
+	/* add metadata */
+	info = xb_builder_node_insert (NULL, "info", NULL);
+	xb_builder_node_insert_text (info, "scope", "system", NULL);
+	xb_builder_node_insert_text (info, "filename", filename, NULL);
+	xb_builder_source_set_info (source, info);
+
+	/* add missing icons as required */
+	fixup1 = xb_builder_fixup_new ("AddIcons",
+				       gs_plugin_appstream_add_icons_cb,
+				       plugin, NULL);
+	xb_builder_fixup_set_max_depth (fixup1, 2);
+	xb_builder_source_add_fixup (source, fixup1);
+
+	/* fix up any legacy installed files */
+	fixup2 = xb_builder_fixup_new ("AppStreamUpgrade2",
+				       gs_plugin_appstream_upgrade_cb,
+				       plugin, NULL);
+	xb_builder_fixup_set_max_depth (fixup2, 3);
+	xb_builder_source_add_fixup (source, fixup2);
+
+	/* add the origin as a search keyword for small repos */
+	fixup3 = xb_builder_fixup_new ("AddOriginKeyword",
+				       gs_plugin_appstream_add_origin_keyword_cb,
+				       plugin, NULL);
+	xb_builder_fixup_set_max_depth (fixup3, 1);
+	xb_builder_source_add_fixup (source, fixup3);
+
+	/* success */
+	xb_builder_import_source (builder, source);
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_appstream_load_appstream (GsPlugin *plugin,
+				    XbBuilder *builder,
+				    const gchar *path,
+				    GCancellable *cancellable,
+				    GError **error)
+{
+	const gchar *fn;
+	g_autoptr(GDir) dir = NULL;
+	g_autoptr(GFile) parent = g_file_new_for_path (path);
+
+	/* parent patch does not exist */
+	if (!g_file_query_exists (parent, cancellable))
+		return TRUE;
+	dir = g_dir_open (path, 0, error);
+	if (dir == NULL)
+		return FALSE;
+	while ((fn = g_dir_read_name (dir)) != NULL) {
+		if (g_str_has_suffix (fn, ".xml") ||
+		    g_str_has_suffix (fn, ".yml") ||
+		    g_str_has_suffix (fn, ".yml.gz") ||
+		    g_str_has_suffix (fn, ".xml.gz")) {
+			g_autofree gchar *filename = g_build_filename (path, fn, NULL);
+			g_autoptr(GError) error_local = NULL;
+			if (!gs_plugin_appstream_load_appstream_fn (plugin,
+								    builder,
+								    filename,
+								    cancellable,
+								    &error_local)) {
+				g_debug ("ignoring %s: %s", filename, error_local->message);
+				continue;
+			}
+		}
+	}
+
+	/* success */
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_appstream_check_silo (GsPlugin *plugin,
+				GCancellable *cancellable,
+				GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *locale;
+	const gchar *test_xml;
+	g_autofree gchar *blobfn = NULL;
+	g_autoptr(XbBuilder) builder = xb_builder_new ();
+	g_autoptr(XbNode) n = NULL;
+	g_autoptr(GFile) file = NULL;
+	g_autoptr(GRWLockReaderLocker) reader_locker = NULL;
+	g_autoptr(GRWLockWriterLocker) writer_locker = NULL;
+	g_autoptr(GPtrArray) parent_appdata = g_ptr_array_new_with_free_func (g_free);
+	g_autoptr(GPtrArray) parent_appstream = g_ptr_array_new_with_free_func (g_free);
+
+
+	reader_locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	/* everything is okay */
+	if (priv->silo != NULL && xb_silo_is_valid (priv->silo))
+		return TRUE;
+	g_clear_pointer (&reader_locker, g_rw_lock_reader_locker_free);
+
+	/* drat! silo needs regenerating */
+	writer_locker = g_rw_lock_writer_locker_new (&priv->silo_lock);
+	g_clear_object (&priv->silo);
+
+	/* verbose profiling */
+	if (g_getenv ("GS_XMLB_VERBOSE") != NULL) {
+		xb_builder_set_profile_flags (builder,
+					      XB_SILO_PROFILE_FLAG_XPATH |
+					      XB_SILO_PROFILE_FLAG_DEBUG);
+	}
+
+	/* add current locales */
+	locale = g_getenv ("GS_SELF_TEST_LOCALE");
+	if (locale == NULL) {
+		const gchar *const *locales = g_get_language_names ();
+		for (guint i = 0; locales[i] != NULL; i++)
+			xb_builder_add_locale (builder, locales[i]);
+	} else {
+		xb_builder_add_locale (builder, locale);
+	}
+
+	/* only when in self test */
+	test_xml = g_getenv ("GS_SELF_TEST_APPSTREAM_XML");
+	if (test_xml != NULL) {
+		g_autoptr(XbBuilderFixup) fixup1 = NULL;
+		g_autoptr(XbBuilderFixup) fixup2 = NULL;
+		g_autoptr(XbBuilderSource) source = xb_builder_source_new ();
+		if (!xb_builder_source_load_xml (source, test_xml,
+						 XB_BUILDER_SOURCE_FLAG_NONE,
+						 error))
+			return FALSE;
+		fixup1 = xb_builder_fixup_new ("AddOriginKeywords",
+					       gs_plugin_appstream_add_origin_keyword_cb,
+					       plugin, NULL);
+		xb_builder_fixup_set_max_depth (fixup1, 1);
+		xb_builder_source_add_fixup (source, fixup1);
+		fixup2 = xb_builder_fixup_new ("AddIcons",
+					       gs_plugin_appstream_add_icons_cb,
+					       plugin, NULL);
+		xb_builder_fixup_set_max_depth (fixup2, 2);
+		xb_builder_source_add_fixup (source, fixup2);
+		xb_builder_import_source (builder, source);
+	} else {
+		/* add search paths */
+		g_ptr_array_add (parent_appstream,
+				 g_build_filename (DATADIR, "app-info", "xmls", NULL));
+		g_ptr_array_add (parent_appstream,
+				 g_build_filename (DATADIR, "app-info", "yaml", NULL));
+		g_ptr_array_add (parent_appdata,
+				 g_build_filename (DATADIR, "appdata", NULL));
+		g_ptr_array_add (parent_appdata,
+				 g_build_filename (DATADIR, "metainfo", NULL));
+		g_ptr_array_add (parent_appstream,
+				 g_build_filename (LOCALSTATEDIR, "cache", "app-info", "xmls", NULL));
+		g_ptr_array_add (parent_appstream,
+				 g_build_filename (LOCALSTATEDIR, "cache", "app-info", "yaml", NULL));
+		g_ptr_array_add (parent_appstream,
+				 g_build_filename (LOCALSTATEDIR, "lib", "app-info", "xmls", NULL));
+		g_ptr_array_add (parent_appstream,
+				 g_build_filename (LOCALSTATEDIR, "lib", "app-info", "yaml", NULL));
+
+		/* Add the normal system directories if the installation prefix
+		 * is different from normal — typically this happens when doing
+		 * development builds. It’s useful to still list the system apps
+		 * during development. */
+		if (g_strcmp0 (DATADIR, "/usr/share") != 0) {
+			g_ptr_array_add (parent_appstream,
+					 g_build_filename ("/usr/share", "app-info", "xmls", NULL));
+			g_ptr_array_add (parent_appstream,
+					 g_build_filename ("/usr/share", "app-info", "yaml", NULL));
+			g_ptr_array_add (parent_appdata,
+					 g_build_filename ("/usr/share", "appdata", NULL));
+			g_ptr_array_add (parent_appdata,
+					 g_build_filename ("/usr/share", "metainfo", NULL));
+		}
+		if (g_strcmp0 (LOCALSTATEDIR, "/var") != 0) {
+			g_ptr_array_add (parent_appstream,
+					 g_build_filename ("/var", "cache", "app-info", "xmls", NULL));
+			g_ptr_array_add (parent_appstream,
+					 g_build_filename ("/var", "cache", "app-info", "yaml", NULL));
+			g_ptr_array_add (parent_appstream,
+					 g_build_filename ("/var", "lib", "app-info", "xmls", NULL));
+			g_ptr_array_add (parent_appstream,
+					 g_build_filename ("/var", "lib", "app-info", "yaml", NULL));
+		}
+
+		/* import all files */
+		for (guint i = 0; i < parent_appstream->len; i++) {
+			const gchar *fn = g_ptr_array_index (parent_appstream, i);
+			if (!gs_plugin_appstream_load_appstream (plugin, builder, fn,
+								 cancellable, error))
+				return FALSE;
+		}
+		for (guint i = 0; i < parent_appdata->len; i++) {
+			const gchar *fn = g_ptr_array_index (parent_appdata, i);
+			if (!gs_plugin_appstream_load_appdata (plugin, builder, fn,
+							       cancellable, error))
+				return FALSE;
+		}
+		if (!gs_plugin_appstream_load_desktop (plugin, builder,
+						       DATADIR "/applications",
+						       cancellable, error)) {
+			return FALSE;
+		}
+		if (g_strcmp0 (DATADIR, "/usr/share") != 0 &&
+		    !gs_plugin_appstream_load_desktop (plugin, builder,
+						       "/usr/share/applications",
+						       cancellable, error)) {
+			return FALSE;
+		}
+	}
+
+	/* regenerate with each minor release */
+	xb_builder_append_guid (builder, PACKAGE_VERSION);
+
+	/* create per-user cache */
+	blobfn = gs_utils_get_cache_filename ("appstream", "components.xmlb",
+					      GS_UTILS_CACHE_FLAG_WRITEABLE,
+					      error);
+	if (blobfn == NULL)
+		return FALSE;
+	file = g_file_new_for_path (blobfn);
+	g_debug ("ensuring %s", blobfn);
+	priv->silo = xb_builder_ensure (builder, file,
+					XB_BUILDER_COMPILE_FLAG_IGNORE_INVALID |
+					XB_BUILDER_COMPILE_FLAG_SINGLE_LANG,
+					NULL, error);
+	if (priv->silo == NULL)
+		return FALSE;
+
+	/* watch all directories too */
+	for (guint i = 0; i < parent_appstream->len; i++) {
+		const gchar *fn = g_ptr_array_index (parent_appstream, i);
+		g_autoptr(GFile) file_tmp = g_file_new_for_path (fn);
+		if (!xb_silo_watch_file (priv->silo, file_tmp, cancellable, error))
+			return FALSE;
+	}
+	for (guint i = 0; i < parent_appdata->len; i++) {
+		const gchar *fn = g_ptr_array_index (parent_appdata, i);
+		g_autoptr(GFile) file_tmp = g_file_new_for_path (fn);
+		if (!xb_silo_watch_file (priv->silo, file_tmp, cancellable, error))
+			return FALSE;
+	}
+
+	/* test we found something */
+	n = xb_silo_query_first (priv->silo, "components/component", NULL);
+	if (n == NULL) {
+		g_warning ("No AppStream data, try 'make install-sample-data' in data/");
+		g_set_error (error,
+			     GS_PLUGIN_ERROR,
+			     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+			     "No AppStream data found");
+		return FALSE;
+	}
+
+	/* success */
+	return TRUE;
+}
+
+gboolean
+gs_plugin_setup (GsPlugin *plugin, GCancellable *cancellable, GError **error)
+{
+	/* set up silo, compiling if required */
+	return gs_plugin_appstream_check_silo (plugin, cancellable, error);
+}
+
+gboolean
+gs_plugin_url_to_app (GsPlugin *plugin,
+		      GsAppList *list,
+		      const gchar *url,
+		      GCancellable *cancellable,
+		      GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autofree gchar *path = NULL;
+	g_autofree gchar *scheme = NULL;
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+	g_autoptr(GsApp) app = NULL;
+	g_autoptr(XbNode) component = NULL;
+
+	/* check silo is valid */
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	/* not us */
+	scheme = gs_utils_get_url_scheme (url);
+	if (g_strcmp0 (scheme, "appstream") != 0)
+		return TRUE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+
+	/* create app */
+	path = gs_utils_get_url_path (url);
+	xpath = g_strdup_printf ("components/component/id[text()='%s']", path);
+	component = xb_silo_query_first (priv->silo, xpath, NULL);
+	if (component == NULL)
+		return TRUE;
+	app = gs_appstream_create_app (plugin, priv->silo, component, error);
+	if (app == NULL)
+		return FALSE;
+	gs_app_set_scope (app, AS_APP_SCOPE_SYSTEM);
+	gs_app_list_add (list, app);
+	return TRUE;
+}
+
+static void
+gs_plugin_appstream_set_compulsory_quirk (GsApp *app, XbNode *component)
+{
+	g_autoptr(GPtrArray) array = NULL;
+	const gchar *current_desktop;
+
+	/*
+	 * Set the core applications for the current desktop that cannot be
+	 * removed.
+	 *
+	 * If XDG_CURRENT_DESKTOP contains ":", indicating that it is made up
+	 * of multiple components per the Desktop Entry Specification, an app
+	 * is compulsory if any of the components in XDG_CURRENT_DESKTOP match
+	 * any value in <compulsory_for_desktops />. In that way,
+	 * "GNOME-Classic:GNOME" shares compulsory apps with GNOME.
+	 *
+	 * As a special case, if the <compulsory_for_desktop /> value contains
+	 * a ":", we match the entire XDG_CURRENT_DESKTOP. This lets people set
+	 * compulsory apps for such compound desktops if they want.
+	 *
+	 */
+	array = xb_node_query (component, "compulsory_for_desktop", 0, NULL);
+	if (array == NULL)
+		return;
+	current_desktop = g_getenv ("XDG_CURRENT_DESKTOP");
+	if (current_desktop != NULL) {
+		g_auto(GStrv) xdg_current_desktops = g_strsplit (current_desktop, ":", 0);
+		for (guint i = 0; i < array->len; i++) {
+			XbNode *n = g_ptr_array_index (array, i);
+			const gchar *tmp = xb_node_get_text (n);
+			/* if the value has a :, check the whole string */
+			if (g_strstr_len (tmp, -1, ":")) {
+				if (g_strcmp0 (current_desktop, tmp) == 0) {
+					gs_app_add_quirk (app, GS_APP_QUIRK_COMPULSORY);
+					break;
+				}
+			/* otherwise check if any element matches this one */
+			} else if (g_strv_contains ((const gchar * const *) xdg_current_desktops, tmp)) {
+				gs_app_add_quirk (app, GS_APP_QUIRK_COMPULSORY);
+				break;
+			}
+		}
+	}
+}
+
+static gboolean
+gs_plugin_appstream_refine_state (GsPlugin *plugin, GsApp *app, GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+	g_autoptr(XbNode) component = NULL;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+
+	xpath = g_strdup_printf ("component/id[text()='%s']", gs_app_get_id (app));
+	component = xb_silo_query_first (priv->silo, xpath, &error_local);
+	if (component == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	gs_app_set_state (app, AS_APP_STATE_INSTALLED);
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_refine_from_id (GsPlugin *plugin,
+			  GsApp *app,
+			  GsPluginRefineFlags flags,
+			  gboolean *found,
+			  GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *id;
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+	g_autoptr(GString) xpath = g_string_new (NULL);
+	g_autoptr(GPtrArray) components = NULL;
+
+	/* not enough info to find */
+	id = gs_app_get_id (app);
+	if (id == NULL)
+		return TRUE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+
+	/* look in AppStream then fall back to AppData */
+	xb_string_append_union (xpath, "components/component/id[text()='%s']/../pkgname/..", id);
+	xb_string_append_union (xpath, "components/component[@type='webapp']/id[text()='%s']/..", id);
+	xb_string_append_union (xpath, "component/id[text()='%s']/..", id);
+	components = xb_silo_query (priv->silo, xpath->str, 0, &error_local);
+	if (components == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < components->len; i++) {
+		XbNode *component = g_ptr_array_index (components, i);
+		if (!gs_appstream_refine_app (plugin, app, priv->silo,
+					      component, flags, error))
+			return FALSE;
+		gs_plugin_appstream_set_compulsory_quirk (app, component);
+	}
+
+	/* if an installed desktop or appdata file exists set to installed */
+	if (gs_app_get_state (app) == AS_APP_STATE_UNKNOWN) {
+		if (!gs_plugin_appstream_refine_state (plugin, app, error))
+			return FALSE;
+	}
+
+	/* success */
+	*found = TRUE;
+	return TRUE;
+}
+
+static gboolean
+gs_plugin_refine_from_pkgname (GsPlugin *plugin,
+			       GsApp *app,
+			       GsPluginRefineFlags flags,
+			       GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	GPtrArray *sources = gs_app_get_sources (app);
+	g_autoptr(GError) error_local = NULL;
+
+	/* not enough info to find */
+	if (sources->len == 0)
+		return TRUE;
+
+	/* find all apps when matching any prefixes */
+	for (guint j = 0; j < sources->len; j++) {
+		const gchar *pkgname = g_ptr_array_index (sources, j);
+		g_autoptr(GRWLockReaderLocker) locker = NULL;
+		g_autoptr(GString) xpath = g_string_new (NULL);
+		g_autoptr(XbNode) component = NULL;
+
+		locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+
+		/* prefer actual apps and then fallback to anything else */
+		xb_string_append_union (xpath, "components/component[@type='desktop']/pkgname[text()='%s']/..", pkgname);
+		xb_string_append_union (xpath, "components/component[@type='console']/pkgname[text()='%s']/..", pkgname);
+		xb_string_append_union (xpath, "components/component[@type='webapp']/pkgname[text()='%s']/..", pkgname);
+		xb_string_append_union (xpath, "components/component/pkgname[text()='%s']/..", pkgname);
+		component = xb_silo_query_first (priv->silo, xpath->str, &error_local);
+		if (component == NULL) {
+			if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+				continue;
+			if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+				continue;
+			g_propagate_error (error, g_steal_pointer (&error_local));
+			return FALSE;
+		}
+		if (!gs_appstream_refine_app (plugin, app, priv->silo, component, flags, error))
+			return FALSE;
+		gs_plugin_appstream_set_compulsory_quirk (app, component);
+	}
+
+	/* success */
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin *plugin,
+		  GsAppList *list,
+		  GsPluginRefineFlags flags,
+		  GCancellable *cancellable,
+		  GError **error)
+{
+	gboolean found = FALSE;
+
+	/* check silo is valid */
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+
+		/* not us */
+		if (gs_app_get_bundle_kind (app) != AS_BUNDLE_KIND_PACKAGE &&
+		    gs_app_get_bundle_kind (app) != AS_BUNDLE_KIND_UNKNOWN)
+			return TRUE;
+
+		/* find by ID then fall back to package name */
+		if (!gs_plugin_refine_from_id (plugin, app, flags, &found, error))
+			return FALSE;
+		if (!found) {
+			if (!gs_plugin_refine_from_pkgname (plugin, app, flags, error))
+				return FALSE;
+		}
+	}
+
+	/* success */
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine_wildcard (GsPlugin *plugin,
+			   GsApp *app,
+			   GsAppList *list,
+			   GsPluginRefineFlags refine_flags,
+			   GCancellable *cancellable,
+			   GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *id;
+	g_autofree gchar *xpath = NULL;
+	g_autoptr(GError) error_local = NULL;
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+	g_autoptr(GPtrArray) components = NULL;
+
+	/* check silo is valid */
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	/* not enough info to find */
+	id = gs_app_get_id (app);
+	if (id == NULL)
+		return TRUE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+
+	/* find all app with package names when matching any prefixes */
+	xpath = g_strdup_printf ("components/component/id[text()='%s']/../pkgname/..", id);
+	components = xb_silo_query (priv->silo, xpath, 0, &error_local);
+	if (components == NULL) {
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+			return TRUE;
+		if (g_error_matches (error_local, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT))
+			return TRUE;
+		g_propagate_error (error, g_steal_pointer (&error_local));
+		return FALSE;
+	}
+	for (guint i = 0; i < components->len; i++) {
+		XbNode *component = g_ptr_array_index (components, i);
+		g_autoptr(GsApp) new = NULL;
+
+		/* new app */
+		new = gs_appstream_create_app (plugin, priv->silo, component, error);
+		if (new == NULL)
+			return FALSE;
+		gs_app_set_scope (new, AS_APP_SCOPE_SYSTEM);
+		gs_app_subsume_metadata (new, app);
+		if (!gs_appstream_refine_app (plugin, new, priv->silo, component,
+					      refine_flags, error))
+			return FALSE;
+		gs_app_list_add (list, new);
+	}
+
+	/* success */
+	return TRUE;
+}
+
+gboolean
+gs_plugin_add_category_apps (GsPlugin *plugin,
+			     GsCategory *category,
+			     GsAppList *list,
+			     GCancellable *cancellable,
+			     GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	return gs_appstream_add_category_apps (plugin,
+					       priv->silo,
+					       category,
+					       list,
+					       cancellable,
+					       error);
+}
+
+gboolean
+gs_plugin_add_search (GsPlugin *plugin,
+		      gchar **values,
+		      GsAppList *list,
+		      GCancellable *cancellable,
+		      GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	return gs_appstream_search (plugin,
+				    priv->silo,
+				    (const gchar * const *) values,
+				    list,
+				    cancellable,
+				    error);
+}
+
+gboolean
+gs_plugin_add_installed (GsPlugin *plugin,
+			 GsAppList *list,
+			 GCancellable *cancellable,
+			 GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+	g_autoptr(GPtrArray) components = NULL;
+
+	/* check silo is valid */
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+
+	/* get all installed appdata files (notice no 'components/' prefix...) */
+	components = xb_silo_query (priv->silo, "component/description/..", 0, NULL);
+	if (components == NULL)
+		return TRUE;
+	for (guint i = 0; i < components->len; i++) {
+		XbNode *component = g_ptr_array_index (components, i);
+		g_autoptr(GsApp) app = gs_appstream_create_app (plugin, priv->silo, component, error);
+		if (app == NULL)
+			return FALSE;
+		gs_app_set_state (app, AS_APP_STATE_INSTALLED);
+		gs_app_set_scope (app, AS_APP_SCOPE_SYSTEM);
+		gs_app_list_add (list, app);
+	}
+	return TRUE;
+}
+
+gboolean
+gs_plugin_add_categories (GsPlugin *plugin,
+			  GPtrArray *list,
+			  GCancellable *cancellable,
+			  GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	return gs_appstream_add_categories (plugin, priv->silo, list,
+					    cancellable, error);
+}
+
+gboolean
+gs_plugin_add_popular (GsPlugin *plugin,
+		       GsAppList *list,
+		       GCancellable *cancellable,
+		       GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	return gs_appstream_add_popular (plugin, priv->silo, list, cancellable, error);
+}
+
+gboolean
+gs_plugin_add_featured (GsPlugin *plugin,
+			GsAppList *list,
+			GCancellable *cancellable,
+			GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	return gs_appstream_add_featured (plugin, priv->silo, list, cancellable, error);
+}
+
+gboolean
+gs_plugin_add_recent (GsPlugin *plugin,
+		      GsAppList *list,
+		      guint64 age,
+		      GCancellable *cancellable,
+		      GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	return gs_appstream_add_recent (plugin, priv->silo, list, age,
+					cancellable, error);
+}
+
+gboolean
+gs_plugin_add_alternates (GsPlugin *plugin,
+			  GsApp *app,
+			  GsAppList *list,
+			  GCancellable *cancellable,
+			  GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_autoptr(GRWLockReaderLocker) locker = NULL;
+
+	if (!gs_plugin_appstream_check_silo (plugin, cancellable, error))
+		return FALSE;
+
+	locker = g_rw_lock_reader_locker_new (&priv->silo_lock);
+	return gs_appstream_add_alternates (plugin, priv->silo, app, list,
+					    cancellable, error);
+}
+
+gboolean
+gs_plugin_refresh (GsPlugin *plugin,
+		   guint cache_age,
+		   GCancellable *cancellable,
+		   GError **error)
+{
+	return gs_plugin_appstream_check_silo (plugin, cancellable, error);
+}
diff --git a/plugins/core/gs-plugin-desktop-categories.c b/plugins/core/gs-plugin-desktop-categories.c
new file mode 100644
index 0000000..f90788d
--- /dev/null
+++ b/plugins/core/gs-plugin-desktop-categories.c
@@ -0,0 +1,106 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2011-2016 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2017 Kalev Lember <klember@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+#include <glib/gi18n.h>
+
+#include "gs-desktop-common.h"
+
+/*
+ * SECTION:
+ * Adds categories from a hardcoded list based on the the desktop menu
+ * specification.
+ */
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	/* need categories */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_BEFORE, "appstream");
+}
+
+gboolean
+gs_plugin_add_categories (GsPlugin *plugin,
+			  GPtrArray *list,
+			  GCancellable *cancellable,
+			  GError **error)
+{
+	const GsDesktopData *msdata;
+	guint i, j, k;
+
+	msdata = gs_desktop_get_data ();
+	for (i = 0; msdata[i].id != NULL; i++) {
+		GsCategory *category;
+		g_autofree gchar *msgctxt = NULL;
+
+		/* add parent category */
+		category = gs_category_new (msdata[i].id);
+		gs_category_set_icon (category, msdata[i].icon);
+		gs_category_set_name (category, gettext (msdata[i].name));
+		gs_category_set_score (category, msdata[i].score);
+		g_ptr_array_add (list, category);
+		msgctxt = g_strdup_printf ("Menu of %s", msdata[i].name);
+
+		/* add subcategories */
+		for (j = 0; msdata[i].mapping[j].id != NULL; j++) {
+			const GsDesktopMap *map = &msdata[i].mapping[j];
+			g_autoptr(GsCategory) sub = gs_category_new (map->id);
+			for (k = 0; map->fdo_cats[k] != NULL; k++)
+				gs_category_add_desktop_group (sub, map->fdo_cats[k]);
+			gs_category_set_name (sub, g_dpgettext2 (GETTEXT_PACKAGE,
+								 msgctxt,
+								 map->name));
+			gs_category_add_child (category, sub);
+		}
+	}
+	return TRUE;
+}
+
+/* most of this time this won't be required, unless the user creates a
+ * GsCategory manually and uses it to get results, for instance in the
+ * overview page or `gnome-software-cmd get-category-apps games/featured` */
+gboolean
+gs_plugin_add_category_apps (GsPlugin *plugin,
+			     GsCategory *category,
+			     GsAppList *list,
+			     GCancellable *cancellable,
+			     GError **error)
+{
+	GPtrArray *desktop_groups;
+	GsCategory *parent;
+	const GsDesktopData *msdata;
+	guint i, j, k;
+
+	/* already set */
+	desktop_groups = gs_category_get_desktop_groups (category);
+	if (desktop_groups->len > 0)
+		return TRUE;
+
+	/* not valid */
+	parent = gs_category_get_parent (category);
+	if (parent == NULL)
+		return TRUE;
+
+	/* find desktop_groups for a parent::child category */
+	msdata = gs_desktop_get_data ();
+	for (i = 0; msdata[i].id != NULL; i++) {
+		if (g_strcmp0 (gs_category_get_id (parent), msdata[i].id) != 0)
+			continue;
+		for (j = 0; msdata[i].mapping[j].id != NULL; j++) {
+			const GsDesktopMap *map = &msdata[i].mapping[j];
+			if (g_strcmp0 (gs_category_get_id (category), map->id) != 0)
+				continue;
+			for (k = 0; map->fdo_cats[k] != NULL; k++)
+				gs_category_add_desktop_group (category, map->fdo_cats[k]);
+		}
+	}
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-desktop-menu-path.c b/plugins/core/gs-plugin-desktop-menu-path.c
new file mode 100644
index 0000000..2ec73b5
--- /dev/null
+++ b/plugins/core/gs-plugin-desktop-menu-path.c
@@ -0,0 +1,111 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2011-2016 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2017 Kalev Lember <klember@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <glib/gi18n.h>
+
+#include <gnome-software.h>
+
+#include "gs-desktop-common.h"
+
+/*
+ * SECTION:
+ * Adds categories from a hardcoded list based on the the desktop menu
+ * specification.
+ */
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	/* need categories */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "appstream");
+}
+
+static gboolean
+_gs_app_has_desktop_group (GsApp *app, const gchar *desktop_group)
+{
+	guint i;
+	g_auto(GStrv) split = g_strsplit (desktop_group, "::", -1);
+	for (i = 0; split[i] != NULL; i++) {
+		if (!gs_app_has_category (app, split[i]))
+			return FALSE;
+	}
+	return TRUE;
+}
+
+/* adds the menu-path for applications */
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	const gchar *strv[] = { "", NULL, NULL };
+	const GsDesktopData *msdata;
+	gboolean found = FALSE;
+	guint i, j, k;
+
+	/* nothing to do here */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_MENU_PATH) == 0)
+		return TRUE;
+	if (gs_app_get_menu_path (app) != NULL)
+		return TRUE;
+
+	/* find a top level category the app has */
+	msdata = gs_desktop_get_data ();
+	for (i = 0; !found && msdata[i].id != NULL; i++) {
+		const GsDesktopData *data = &msdata[i];
+		for (j = 0; !found && data->mapping[j].id != NULL; j++) {
+			const GsDesktopMap *map = &data->mapping[j];
+			g_autofree gchar *msgctxt = NULL;
+
+			if (g_strcmp0 (map->id, "all") == 0)
+				continue;
+			if (g_strcmp0 (map->id, "featured") == 0)
+				continue;
+			msgctxt = g_strdup_printf ("Menu of %s", data->name);
+			for (k = 0; !found && map->fdo_cats[k] != NULL; k++) {
+				const gchar *tmp = msdata[i].mapping[j].fdo_cats[k];
+				if (_gs_app_has_desktop_group (app, tmp)) {
+					strv[0] = g_dgettext (GETTEXT_PACKAGE, msdata[i].name);
+					strv[1] = g_dpgettext2 (GETTEXT_PACKAGE, msgctxt,
+					                        msdata[i].mapping[j].name);
+					found = TRUE;
+					break;
+				}
+			}
+		}
+	}
+
+	/* always set something to avoid keep searching for this */
+	gs_app_set_menu_path (app, (gchar **) strv);
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	/* nothing to do here */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_MENU_PATH) == 0)
+		return TRUE;
+
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-generic-updates.c b/plugins/core/gs-plugin-generic-updates.c
new file mode 100644
index 0000000..8ca56e1
--- /dev/null
+++ b/plugins/core/gs-plugin-generic-updates.c
@@ -0,0 +1,106 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2016 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <glib/gi18n.h>
+#include <gnome-software.h>
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "appstream");
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "packagekit-refine");
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "rpm-ostree");
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_BEFORE, "icons");
+}
+
+static gboolean
+gs_plugin_generic_updates_merge_os_update (GsApp *app)
+{
+	/* this is only for grouping system-installed packages */
+	if (gs_app_get_bundle_kind (app) != AS_BUNDLE_KIND_PACKAGE ||
+	    gs_app_get_scope (app) != AS_APP_SCOPE_SYSTEM)
+		return FALSE;
+
+	if (gs_app_get_kind (app) == AS_APP_KIND_GENERIC)
+		return TRUE;
+	if (gs_app_get_kind (app) == AS_APP_KIND_SOURCE)
+		return TRUE;
+
+	return FALSE;
+}
+
+static GsApp *
+gs_plugin_generic_updates_get_os_update (GsPlugin *plugin)
+{
+	GsApp *app;
+	const gchar *id = "org.gnome.Software.OsUpdate";
+	g_autoptr(AsIcon) ic = NULL;
+
+	/* create new */
+	app = gs_app_new (id);
+	gs_app_add_quirk (app, GS_APP_QUIRK_IS_PROXY);
+	gs_app_set_management_plugin (app, "");
+	gs_app_set_kind (app, AS_APP_KIND_OS_UPDATE);
+	gs_app_set_state (app, AS_APP_STATE_UPDATABLE_LIVE);
+	gs_app_set_name (app,
+			 GS_APP_QUALITY_NORMAL,
+			 /* TRANSLATORS: this is a group of updates that are not
+			  * packages and are not shown in the main list */
+			 _("OS Updates"));
+	gs_app_set_summary (app,
+			    GS_APP_QUALITY_NORMAL,
+			    /* TRANSLATORS: this is a longer description of the
+			     * "OS Updates" string */
+			    _("Includes performance, stability and security improvements."));
+	gs_app_set_description (app,
+				GS_APP_QUALITY_NORMAL,
+				gs_app_get_summary (app));
+	ic = as_icon_new ();
+	as_icon_set_kind (ic, AS_ICON_KIND_STOCK);
+	as_icon_set_name (ic, "software-update-available-symbolic");
+	gs_app_add_icon (app, ic);
+	return app;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin *plugin,
+		  GsAppList *list,
+		  GsPluginRefineFlags flags,
+		  GCancellable *cancellable,
+		  GError **error)
+{
+	g_autoptr(GsApp) app = NULL;
+	g_autoptr(GsAppList) os_updates = gs_app_list_new ();
+
+	/* not from get_updates() */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_UPDATE_DETAILS) == 0)
+		return TRUE;
+
+	/* do we have any packages left that are not apps? */
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app_tmp = gs_app_list_index (list, i);
+		if (gs_app_has_quirk (app_tmp, GS_APP_QUIRK_IS_WILDCARD))
+			continue;
+		if (gs_plugin_generic_updates_merge_os_update (app_tmp))
+			gs_app_list_add (os_updates, app_tmp);
+	}
+	if (gs_app_list_length (os_updates) == 0)
+		return TRUE;
+
+	/* create new meta object */
+	app = gs_plugin_generic_updates_get_os_update (plugin);
+	for (guint i = 0; i < gs_app_list_length (os_updates); i++) {
+		GsApp *app_tmp = gs_app_list_index (os_updates, i);
+		gs_app_add_related (app, app_tmp);
+		gs_app_list_remove (list, app_tmp);
+	}
+	gs_app_list_add (list, app);
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-hardcoded-blocklist.c b/plugins/core/gs-plugin-hardcoded-blocklist.c
new file mode 100644
index 0000000..f993535
--- /dev/null
+++ b/plugins/core/gs-plugin-hardcoded-blocklist.c
@@ -0,0 +1,78 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2016 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <fnmatch.h>
+#include <gnome-software.h>
+
+/*
+ * SECTION:
+ * Blocklists some applications based on a hardcoded list.
+ */
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	/* need ID */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "appstream");
+}
+
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	guint i;
+	const gchar *app_globs[] = {
+		"freeciv-server.desktop",
+		"links.desktop",
+		"nm-connection-editor.desktop",
+		"plank.desktop",
+		"*release-notes*.desktop",
+		"*Release_Notes*.desktop",
+		"Rodent-*.desktop",
+		"rygel-preferences.desktop",
+		"system-config-keyboard.desktop",
+		"tracker-preferences.desktop",
+		"Uninstall*.desktop",
+		"wine-*.desktop",
+		NULL };
+
+	/* not set yet */
+	if (gs_app_get_id (app) == NULL)
+		return TRUE;
+
+	/* search */
+	for (i = 0; app_globs[i] != NULL; i++) {
+		if (fnmatch (app_globs[i], gs_app_get_id (app), 0) == 0) {
+			gs_app_add_quirk (app, GS_APP_QUIRK_HIDE_EVERYWHERE);
+			break;
+		}
+	}
+
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-hardcoded-popular.c b/plugins/core/gs-plugin-hardcoded-popular.c
new file mode 100644
index 0000000..2431274
--- /dev/null
+++ b/plugins/core/gs-plugin-hardcoded-popular.c
@@ -0,0 +1,66 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2016 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	/* let appstream add applications first */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "appstream");
+}
+
+gboolean
+gs_plugin_add_popular (GsPlugin *plugin,
+		       GsAppList *list,
+		       GCancellable *cancellable,
+		       GError **error)
+{
+	guint i;
+	const gchar *apps[] = {
+		"org.gnome.Builder.desktop",
+		"org.gnome.Calculator.desktop",
+		"org.gnome.clocks.desktop",
+		"org.gnome.Dictionary.desktop",
+		"org.gnome.Documents.desktop",
+		"org.gnome.Evince",
+		"org.gnome.gedit.desktop",
+		"org.gnome.Maps.desktop",
+		"org.gnome.Weather",
+		NULL };
+
+	/* we've already got enough popular apps */
+	if (gs_app_list_length (list) >= 9)
+		return TRUE;
+
+	/* just add all */
+	g_debug ("using hardcoded as only %u apps", gs_app_list_length (list));
+	for (i = 0; apps[i] != NULL; i++) {
+		g_autoptr(GsApp) app = NULL;
+
+		/* look in the cache */
+		app = gs_plugin_cache_lookup (plugin, apps[i]);
+		if (app != NULL) {
+			gs_app_list_add (list, app);
+			continue;
+		}
+
+		/* create new */
+		app = gs_app_new (apps[i]);
+		gs_app_add_quirk (app, GS_APP_QUIRK_IS_WILDCARD);
+		gs_app_set_metadata (app, "GnomeSoftware::Creator",
+				     gs_plugin_get_name (plugin));
+		gs_app_list_add (list, app);
+
+		/* save in the cache */
+		gs_plugin_cache_add (plugin, apps[i], app);
+	}
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-icons.c b/plugins/core/gs-plugin-icons.c
new file mode 100644
index 0000000..112f1d2
--- /dev/null
+++ b/plugins/core/gs-plugin-icons.c
@@ -0,0 +1,348 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2014 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2015 Kalev Lember <klember@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <string.h>
+
+#include <gnome-software.h>
+
+/*
+ * SECTION:
+ * Loads remote icons and converts them into local cached ones.
+ *
+ * It is provided so that each plugin handling icons does not
+ * have to handle the download and caching functionality.
+ */
+
+struct GsPluginData {
+	GtkIconTheme		*icon_theme;
+	GMutex			 icon_theme_lock;
+	GHashTable		*icon_theme_paths;
+};
+
+static void gs_plugin_icons_add_theme_path (GsPlugin *plugin, const gchar *path);
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_alloc_data (plugin, sizeof(GsPluginData));
+	const gchar *test_search_path;
+
+	priv->icon_theme = gtk_icon_theme_new ();
+	gtk_icon_theme_set_screen (priv->icon_theme, gdk_screen_get_default ());
+	priv->icon_theme_paths = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+	g_mutex_init (&priv->icon_theme_lock);
+
+	test_search_path = g_getenv ("GS_SELF_TEST_ICON_THEME_PATH");
+	if (test_search_path != NULL) {
+		g_auto(GStrv) dirs = g_strsplit (test_search_path, ":", -1);
+
+		/* add_theme_path() prepends, so we have to iterate in reverse to preserve order */
+		for (gsize i = g_strv_length (dirs); i > 0; i--)
+			gs_plugin_icons_add_theme_path (plugin, dirs[i - 1]);
+	}
+
+	/* needs remote icons downloaded */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "appstream");
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "epiphany");
+}
+
+void
+gs_plugin_destroy (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_object_unref (priv->icon_theme);
+	g_hash_table_unref (priv->icon_theme_paths);
+	g_mutex_clear (&priv->icon_theme_lock);
+}
+
+static gboolean
+gs_plugin_icons_download (GsPlugin *plugin,
+			  const gchar *uri,
+			  const gchar *filename,
+			  GError **error)
+{
+	guint status_code;
+	g_autoptr(GdkPixbuf) pixbuf_new = NULL;
+	g_autoptr(GdkPixbuf) pixbuf = NULL;
+	g_autoptr(GInputStream) stream = NULL;
+	g_autoptr(SoupMessage) msg = NULL;
+
+	/* create the GET data */
+	msg = soup_message_new (SOUP_METHOD_GET, uri);
+	if (msg == NULL) {
+		g_set_error (error,
+			     GS_PLUGIN_ERROR,
+			     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+			     "%s is not a valid URL", uri);
+		return FALSE;
+	}
+
+	/* set sync request */
+	status_code = soup_session_send_message (gs_plugin_get_soup_session (plugin), msg);
+	if (status_code != SOUP_STATUS_OK) {
+		g_set_error (error,
+			     GS_PLUGIN_ERROR,
+			     GS_PLUGIN_ERROR_DOWNLOAD_FAILED,
+			     "Failed to download icon %s: %s",
+			     uri, soup_status_get_phrase (status_code));
+		return FALSE;
+	}
+
+	/* we're assuming this is a 64x64 png file, resize if not */
+	stream = g_memory_input_stream_new_from_data (msg->response_body->data,
+						      msg->response_body->length,
+						      NULL);
+	pixbuf = gdk_pixbuf_new_from_stream (stream, NULL, error);
+	if (pixbuf == NULL) {
+		gs_utils_error_convert_gdk_pixbuf (error);
+		return FALSE;
+	}
+	if (gdk_pixbuf_get_height (pixbuf) == 64 &&
+	    gdk_pixbuf_get_width (pixbuf) == 64) {
+		pixbuf_new = g_object_ref (pixbuf);
+	} else {
+		pixbuf_new = gdk_pixbuf_scale_simple (pixbuf, 64, 64,
+						      GDK_INTERP_BILINEAR);
+	}
+
+	/* write file */
+	if (!gdk_pixbuf_save (pixbuf_new, filename, "png", error, NULL)) {
+		gs_utils_error_convert_gdk_pixbuf (error);
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static GdkPixbuf *
+gs_plugin_icons_load_local (GsPlugin *plugin, AsIcon *icon, GError **error)
+{
+	GdkPixbuf *pixbuf;
+	gint size;
+	if (as_icon_get_filename (icon) == NULL) {
+		g_set_error_literal (error,
+				     GS_PLUGIN_ERROR,
+				     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+				     "icon has no filename");
+		return NULL;
+	}
+	size = (gint) (64 * gs_plugin_get_scale (plugin));
+	pixbuf = gdk_pixbuf_new_from_file_at_size (as_icon_get_filename (icon),
+						   size, size, error);
+	if (pixbuf == NULL) {
+		gs_utils_error_convert_gdk_pixbuf (error);
+		return NULL;
+	}
+	return pixbuf;
+}
+
+static gchar *
+gs_plugin_icons_get_cache_fn (AsIcon *icon)
+{
+	g_autofree gchar *basename = NULL;
+	g_autofree gchar *checksum = NULL;
+	checksum = g_compute_checksum_for_string (G_CHECKSUM_SHA1,
+						  as_icon_get_url (icon),
+						  -1);
+	basename = g_path_get_basename (as_icon_get_url (icon));
+	return g_strdup_printf ("%s-%s", checksum, basename);
+}
+
+static GdkPixbuf *
+gs_plugin_icons_load_remote (GsPlugin *plugin, AsIcon *icon, GError **error)
+{
+	const gchar *fn;
+	gchar *found;
+
+	/* not applicable for remote */
+	if (as_icon_get_url (icon) == NULL) {
+		g_set_error_literal (error,
+				     GS_PLUGIN_ERROR,
+				     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+				     "icon has no URL");
+		return NULL;
+	}
+
+	/* set cache filename if not already set */
+	if (as_icon_get_filename (icon) == NULL) {
+		g_autofree gchar *fn_cache = NULL;
+		g_autofree gchar *fn_basename = NULL;
+
+		/* use a hash-prefixed filename to avoid cache clashes */
+		fn_basename = gs_plugin_icons_get_cache_fn (icon);
+		fn_cache = gs_utils_get_cache_filename ("icons",
+							fn_basename,
+							GS_UTILS_CACHE_FLAG_WRITEABLE,
+							error);
+		if (fn_cache == NULL)
+			return NULL;
+		as_icon_set_filename (icon, fn_cache);
+	}
+
+	/* already in cache */
+	if (g_file_test (as_icon_get_filename (icon), G_FILE_TEST_EXISTS))
+		return gs_plugin_icons_load_local (plugin, icon, error);
+
+	/* a REMOTE that's really LOCAL */
+	if (g_str_has_prefix (as_icon_get_url (icon), "file://")) {
+		as_icon_set_filename (icon, as_icon_get_url (icon) + 7);
+		as_icon_set_kind (icon, AS_ICON_KIND_LOCAL);
+		return gs_plugin_icons_load_local (plugin, icon, error);
+	}
+
+	/* convert filename from jpg to png */
+	fn = as_icon_get_filename (icon);
+	found = g_strstr_len (fn, -1, ".jpg");
+	if (found != NULL)
+		memcpy (found, ".png", 4);
+
+	/* create runtime dir and download */
+	if (!gs_mkdir_parent (fn, error))
+		return NULL;
+	if (!gs_plugin_icons_download (plugin, as_icon_get_url (icon), fn, error))
+		return NULL;
+	as_icon_set_kind (icon, AS_ICON_KIND_LOCAL);
+	return gs_plugin_icons_load_local (plugin, icon, error);
+}
+
+static void
+gs_plugin_icons_add_theme_path (GsPlugin *plugin, const gchar *path)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	if (path == NULL)
+		return;
+	if (!g_hash_table_contains (priv->icon_theme_paths, path)) {
+		gtk_icon_theme_prepend_search_path (priv->icon_theme, path);
+		g_hash_table_add (priv->icon_theme_paths, g_strdup (path));
+	}
+}
+
+static GdkPixbuf *
+gs_plugin_icons_load_stock (GsPlugin *plugin, AsIcon *icon, GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	GdkPixbuf *pixbuf;
+	gint size;
+	g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->icon_theme_lock);
+
+	/* required */
+	if (as_icon_get_name (icon) == NULL) {
+		g_set_error_literal (error,
+				     GS_PLUGIN_ERROR,
+				     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+				     "icon has no name");
+		return NULL;
+	}
+	gs_plugin_icons_add_theme_path (plugin, as_icon_get_prefix (icon));
+	size = (gint) (64 * gs_plugin_get_scale (plugin));
+	pixbuf = gtk_icon_theme_load_icon (priv->icon_theme,
+					   as_icon_get_name (icon),
+					   size,
+					   GTK_ICON_LOOKUP_USE_BUILTIN |
+					   GTK_ICON_LOOKUP_FORCE_SIZE,
+					   error);
+	if (pixbuf == NULL) {
+		gs_utils_error_convert_gdk_pixbuf (error);
+		return NULL;
+	}
+	return pixbuf;
+}
+
+static GdkPixbuf *
+gs_plugin_icons_load_cached (GsPlugin *plugin, AsIcon *icon, GError **error)
+{
+	if (!as_icon_load (icon, AS_ICON_LOAD_FLAG_SEARCH_SIZE, error)) {
+		gs_utils_error_convert_gdk_pixbuf (error);
+		gs_utils_error_convert_appstream (error);
+		return NULL;
+	}
+	return g_object_ref (as_icon_get_pixbuf (icon));
+}
+
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	GPtrArray *icons;
+	guint i;
+
+	/* not required */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_ICON) == 0)
+		return TRUE;
+
+	/* invalid */
+	if (gs_app_get_pixbuf (app) != NULL)
+		return TRUE;
+
+	/* process all icons */
+	icons = gs_app_get_icons (app);
+	for (i = 0; i < icons->len; i++) {
+		AsIcon *icon = g_ptr_array_index (icons, i);
+		g_autoptr(GdkPixbuf) pixbuf = NULL;
+		g_autoptr(GError) error_local = NULL;
+
+		/* handle different icon types */
+		switch (as_icon_get_kind (icon)) {
+		case AS_ICON_KIND_LOCAL:
+			pixbuf = gs_plugin_icons_load_local (plugin, icon, &error_local);
+			break;
+		case AS_ICON_KIND_STOCK:
+			pixbuf = gs_plugin_icons_load_stock (plugin, icon, &error_local);
+			break;
+		case AS_ICON_KIND_REMOTE:
+			pixbuf = gs_plugin_icons_load_remote (plugin, icon, &error_local);
+			break;
+		case AS_ICON_KIND_CACHED:
+			pixbuf = gs_plugin_icons_load_cached (plugin, icon, &error_local);
+			break;
+		default:
+			g_set_error (&error_local,
+				     GS_PLUGIN_ERROR,
+				     GS_PLUGIN_ERROR_NOT_SUPPORTED,
+				     "icon kind '%s' unknown",
+				     as_icon_kind_to_string (as_icon_get_kind (icon)));
+			break;
+		}
+		if (pixbuf != NULL) {
+			gs_app_set_pixbuf (app, pixbuf);
+			break;
+		}
+
+		/* we failed, but keep going */
+		g_debug ("failed to load icon for %s: %s",
+			 gs_app_get_id (app),
+			 error_local->message);
+	}
+
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	/* nothing to do here */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_ICON) == 0)
+		return TRUE;
+
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-key-colors-metadata.c b/plugins/core/gs-plugin-key-colors-metadata.c
new file mode 100644
index 0000000..2b8fa21
--- /dev/null
+++ b/plugins/core/gs-plugin-key-colors-metadata.c
@@ -0,0 +1,89 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2017 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "key-colors");
+}
+
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	GPtrArray *key_colors;
+	const gchar *keys[] = {
+		"GnomeSoftware::AppTile-css",
+		"GnomeSoftware::FeatureTile-css",
+		"GnomeSoftware::UpgradeBanner-css",
+		NULL };
+
+	/* not set */
+	key_colors = gs_app_get_key_colors (app);
+	if (key_colors->len == 0)
+		return TRUE;
+
+	/* rewrite URIs */
+	for (guint i = 0; keys[i] != NULL; i++) {
+		const gchar *css;
+		g_autoptr(GString) css_new = NULL;
+
+		/* metadata is not set */
+		css = gs_app_get_metadata_item (app, keys[i]);
+		if (css == NULL)
+			continue;
+		if (g_strstr_len (css, -1, "@keycolor") == NULL)
+			continue;
+
+		/* replace key color values */
+		css_new = g_string_new (css);
+		for (guint j = 0; j < key_colors->len; j++) {
+			GdkRGBA *color = g_ptr_array_index (key_colors, j);
+			g_autofree gchar *key = NULL;
+			g_autofree gchar *value = NULL;
+			key = g_strdup_printf ("@keycolor-%02u@", j);
+			value = g_strdup_printf ("rgb(%.0f,%.0f,%.0f)",
+						 color->red * 255.f,
+						 color->green * 255.f,
+						 color->blue * 255.f);
+			as_utils_string_replace (css_new, key, value);
+		}
+
+		/* only replace if it's different */
+		if (g_strcmp0 (css, css_new->str) != 0) {
+			gs_app_set_metadata (app, keys[i], NULL);
+			gs_app_set_metadata (app, keys[i], css_new->str);
+		}
+
+	}
+
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-key-colors.c b/plugins/core/gs-plugin-key-colors.c
new file mode 100644
index 0000000..08bf7a8
--- /dev/null
+++ b/plugins/core/gs-plugin-key-colors.c
@@ -0,0 +1,195 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2016 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	/* need icon */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "icons");
+}
+
+typedef struct {
+	guint8		 R;
+	guint8		 G;
+	guint8		 B;
+} CdColorRGB8;
+
+static guint32
+cd_color_rgb8_to_uint32 (CdColorRGB8 *rgb)
+{
+	return (guint32) rgb->R |
+		(guint32) rgb->G << 8 |
+		(guint32) rgb->B << 16;
+}
+
+typedef struct {
+	GdkRGBA		color;
+	guint		cnt;
+} GsColorBin;
+
+static gint
+gs_color_bin_sort_cb (gconstpointer a, gconstpointer b)
+{
+	GsColorBin *s1 = (GsColorBin *) a;
+	GsColorBin *s2 = (GsColorBin *) b;
+	if (s1->cnt < s2->cnt)
+		return 1;
+	if (s1->cnt > s2->cnt)
+		return -1;
+	return 0;
+}
+
+/* convert range of 0..255 to 0..1 */
+static inline gdouble
+_convert_from_rgb8 (guchar val)
+{
+	return (gdouble) val / 255.f;
+}
+
+static void
+gs_plugin_key_colors_set_for_pixbuf (GsApp *app, GdkPixbuf *pb, guint number)
+{
+	gint rowstride, n_channels;
+	gint x, y, width, height;
+	guchar *pixels, *p;
+	guint bin_size = 200;
+	guint i;
+	guint number_of_bins;
+
+	/* go through each pixel */
+	n_channels = gdk_pixbuf_get_n_channels (pb);
+	rowstride = gdk_pixbuf_get_rowstride (pb);
+	pixels = gdk_pixbuf_get_pixels (pb);
+	width = gdk_pixbuf_get_width (pb);
+	height = gdk_pixbuf_get_height (pb);
+
+	for (bin_size = 250; bin_size > 0; bin_size -= 2) {
+		g_autoptr(GHashTable) hash = NULL;
+		hash = g_hash_table_new_full (g_direct_hash,  g_direct_equal,
+					      NULL, g_free);
+		for (y = 0; y < height; y++) {
+			for (x = 0; x < width; x++) {
+				CdColorRGB8 tmp;
+				GsColorBin *s;
+				gpointer key;
+
+				/* disregard any with alpha */
+				p = pixels + y * rowstride + x * n_channels;
+				if (p[3] != 255)
+					continue;
+
+				/* find in cache */
+				tmp.R = (guint8) (p[0] / bin_size);
+				tmp.G = (guint8) (p[1] / bin_size);
+				tmp.B = (guint8) (p[2] / bin_size);
+				key = GUINT_TO_POINTER (cd_color_rgb8_to_uint32 (&tmp));
+				s = g_hash_table_lookup (hash, key);
+				if (s != NULL) {
+					s->color.red += _convert_from_rgb8 (p[0]);
+					s->color.green += _convert_from_rgb8 (p[1]);
+					s->color.blue += _convert_from_rgb8 (p[2]);
+					s->cnt++;
+					continue;
+				}
+
+				/* add to hash table */
+				s = g_new0 (GsColorBin, 1);
+				s->color.red = _convert_from_rgb8 (p[0]);
+				s->color.green = _convert_from_rgb8 (p[1]);
+				s->color.blue = _convert_from_rgb8 (p[2]);
+				s->color.alpha = 1.0;
+				s->cnt = 1;
+				g_hash_table_insert (hash, key, s);
+			}
+		}
+
+		number_of_bins = g_hash_table_size (hash);
+//		g_debug ("number of colors: %i", number_of_bins);
+		if (number_of_bins >= number) {
+			g_autoptr(GList) values = NULL;
+
+			/* order by most popular */
+			values = g_hash_table_get_values (hash);
+			values = g_list_sort (values, gs_color_bin_sort_cb);
+			for (GList *l = values; l != NULL; l = l->next) {
+				GsColorBin *s = l->data;
+				g_autofree GdkRGBA *color = g_new0 (GdkRGBA, 1);
+				color->red = s->color.red / s->cnt;
+				color->green = s->color.green / s->cnt;
+				color->blue = s->color.blue / s->cnt;
+				gs_app_add_key_color (app, color);
+			}
+			return;
+		}
+	}
+
+	/* the algorithm failed, so just return a monochrome ramp */
+	for (i = 0; i < 3; i++) {
+		g_autofree GdkRGBA *color = g_new0 (GdkRGBA, 1);
+		color->red = (gdouble) i / 3.f;
+		color->green = color->red;
+		color->blue = color->red;
+		color->alpha = 1.0f;
+		gs_app_add_key_color (app, color);
+	}
+}
+
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	GdkPixbuf *pb;
+	g_autoptr(GdkPixbuf) pb_small = NULL;
+
+	/* add a rating */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_KEY_COLORS) == 0)
+		return TRUE;
+
+	/* already set */
+	if (gs_app_get_key_colors(app)->len > 0)
+		return TRUE;
+
+	/* no pixbuf */
+	pb = gs_app_get_pixbuf (app);
+	if (pb == NULL) {
+		g_debug ("no pixbuf, so no key colors");
+		return TRUE;
+	}
+
+	/* get a list of key colors */
+	pb_small = gdk_pixbuf_scale_simple (pb, 32, 32, GDK_INTERP_BILINEAR);
+	gs_plugin_key_colors_set_for_pixbuf (app, pb_small, 10);
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	/* nothing to do here */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_KEY_COLORS) == 0)
+		return TRUE;
+
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-os-release.c b/plugins/core/gs-plugin-os-release.c
new file mode 100644
index 0000000..3ae902f
--- /dev/null
+++ b/plugins/core/gs-plugin-os-release.c
@@ -0,0 +1,112 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2016 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+
+struct GsPluginData {
+	GsApp			*app_system;
+};
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_alloc_data (plugin, sizeof(GsPluginData));
+	priv->app_system = gs_app_new ("system");
+	gs_app_set_kind (priv->app_system, AS_APP_KIND_OS_UPGRADE);
+	gs_app_set_state (priv->app_system, AS_APP_STATE_INSTALLED);
+}
+
+void
+gs_plugin_destroy (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_object_unref (priv->app_system);
+}
+
+gboolean
+gs_plugin_setup (GsPlugin *plugin, GCancellable *cancellable, GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *cpe_name;
+	const gchar *home_url;
+	const gchar *name;
+	const gchar *version;
+	g_autoptr(GsOsRelease) os_release = NULL;
+
+	/* parse os-release, wherever it may be */
+	os_release = gs_os_release_new (error);
+	if (os_release == NULL)
+		return FALSE;
+	cpe_name = gs_os_release_get_cpe_name (os_release);
+	if (cpe_name != NULL)
+		gs_app_set_metadata (priv->app_system, "GnomeSoftware::CpeName", cpe_name);
+	name = gs_os_release_get_name (os_release);
+	if (name != NULL)
+		gs_app_set_name (priv->app_system, GS_APP_QUALITY_LOWEST, name);
+	version = gs_os_release_get_version_id (os_release);
+	if (version != NULL)
+		gs_app_set_version (priv->app_system, version);
+
+	/* use libsoup to convert a URL */
+	home_url = gs_os_release_get_home_url (os_release);
+	if (home_url != NULL) {
+		g_autoptr(SoupURI) uri = NULL;
+
+		/* homepage */
+		gs_app_set_url (priv->app_system, AS_URL_KIND_HOMEPAGE, home_url);
+
+		/* build ID from the reverse-DNS URL and the name version */
+		uri = soup_uri_new (home_url);
+		if (uri != NULL) {
+			g_auto(GStrv) split = NULL;
+			const gchar *home_host = soup_uri_get_host (uri);
+			split = g_strsplit_set (home_host, ".", -1);
+			if (g_strv_length (split) >= 2) {
+				g_autofree gchar *id = NULL;
+				id = g_strdup_printf ("%s.%s.%s-%s",
+						      split[1],
+						      split[0],
+						      name,
+						      version);
+				gs_app_set_id (priv->app_system, id);
+			}
+		}
+	}
+
+	/* success */
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine_wildcard (GsPlugin *plugin,
+			   GsApp *app,
+			   GsAppList *list,
+			   GsPluginRefineFlags flags,
+			   GCancellable *cancellable,
+			   GError **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+
+	/* match meta-id */
+	if (g_strcmp0 (gs_app_get_id (app), "system") == 0) {
+		/* copy over interesting metadata */
+		if (gs_app_get_install_date (app) != 0 &&
+		    gs_app_get_install_date (priv->app_system) == 0) {
+			gs_app_set_install_date (priv->app_system,
+			                         gs_app_get_install_date (app));
+		}
+
+		gs_app_list_add (list, priv->app_system);
+		return TRUE;
+	}
+
+	/* success */
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-provenance-license.c b/plugins/core/gs-plugin-provenance-license.c
new file mode 100644
index 0000000..1103a6d
--- /dev/null
+++ b/plugins/core/gs-plugin-provenance-license.c
@@ -0,0 +1,152 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2016 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2016 Matthias Klumpp <mak@debian.org>
+ * Copyright (C) 2018 Kalev Lember <klember@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+
+/*
+ * SECTION:
+ * Marks the application as Free Software if it comes from an origin
+ * that is recognized as being DFSGish-free.
+ */
+
+struct GsPluginData {
+	GSettings		*settings;
+	gchar			**sources;
+	gchar			*license_id;
+};
+
+static gchar **
+gs_plugin_provenance_license_get_sources (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *tmp;
+
+	tmp = g_getenv ("GS_SELF_TEST_PROVENANCE_LICENSE_SOURCES");
+	if (tmp != NULL) {
+		g_debug ("using custom provenance_license sources of %s", tmp);
+		return g_strsplit (tmp, ",", -1);
+	}
+	return g_settings_get_strv (priv->settings, "free-repos");
+}
+
+static gchar *
+gs_plugin_provenance_license_get_id (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *tmp;
+	g_autofree gchar *url = NULL;
+
+	tmp = g_getenv ("GS_SELF_TEST_PROVENANCE_LICENSE_URL");
+	if (tmp != NULL) {
+		g_debug ("using custom license generic sources of %s", tmp);
+		url = g_strdup (tmp);
+	} else {
+		url = g_settings_get_string (priv->settings, "free-repos-url");
+		if (url == NULL)
+			return g_strdup ("LicenseRef-free");
+	}
+	return g_strdup_printf ("LicenseRef-free=%s", url);
+}
+
+static void
+gs_plugin_provenance_license_changed_cb (GSettings *settings,
+					 const gchar *key,
+					 GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	if (g_strcmp0 (key, "free-repos") == 0) {
+		g_strfreev (priv->sources);
+		priv->sources = gs_plugin_provenance_license_get_sources (plugin);
+	}
+	if (g_strcmp0 (key, "free-repos-url") == 0) {
+		g_free (priv->license_id);
+		priv->license_id = gs_plugin_provenance_license_get_id (plugin);
+	}
+}
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_alloc_data (plugin, sizeof(GsPluginData));
+	priv->settings = g_settings_new ("org.gnome.software");
+	g_signal_connect (priv->settings, "changed",
+			  G_CALLBACK (gs_plugin_provenance_license_changed_cb), plugin);
+	priv->sources = gs_plugin_provenance_license_get_sources (plugin);
+	priv->license_id = gs_plugin_provenance_license_get_id (plugin);
+
+	/* need this set */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "provenance");
+}
+
+void
+gs_plugin_destroy (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_strfreev (priv->sources);
+	g_free (priv->license_id);
+	g_object_unref (priv->settings);
+}
+
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *origin;
+
+	/* not required */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_LICENSE) == 0)
+		return TRUE;
+
+	/* no provenance */
+	if (!gs_app_has_quirk (app, GS_APP_QUIRK_PROVENANCE))
+		return TRUE;
+
+	/* nothing to search */
+	if (priv->sources == NULL || priv->sources[0] == NULL)
+		return TRUE;
+
+	/* simple case */
+	origin = gs_app_get_origin (app);
+	if (origin != NULL && gs_utils_strv_fnmatch (priv->sources, origin))
+		gs_app_set_license (app, GS_APP_QUALITY_NORMAL, priv->license_id);
+
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+
+	/* nothing to do here */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_LICENSE) == 0)
+		return TRUE;
+	/* nothing to search */
+	if (priv->sources == NULL || priv->sources[0] == NULL)
+		return TRUE;
+
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-provenance.c b/plugins/core/gs-plugin-provenance.c
new file mode 100644
index 0000000..03a854b
--- /dev/null
+++ b/plugins/core/gs-plugin-provenance.c
@@ -0,0 +1,141 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2015-2016 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2018 Kalev Lember <klember@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <gnome-software.h>
+
+/*
+ * SECTION:
+ * Sets the package provenance to TRUE if installed by an official
+ * software source.
+ */
+
+struct GsPluginData {
+	GSettings		*settings;
+	gchar			**sources;
+};
+
+static gchar **
+gs_plugin_provenance_get_sources (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *tmp;
+	tmp = g_getenv ("GS_SELF_TEST_PROVENANCE_SOURCES");
+	if (tmp != NULL) {
+		g_debug ("using custom provenance sources of %s", tmp);
+		return g_strsplit (tmp, ",", -1);
+	}
+	return g_settings_get_strv (priv->settings, "official-repos");
+}
+
+static void
+gs_plugin_provenance_settings_changed_cb (GSettings *settings,
+					  const gchar *key,
+					  GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	if (g_strcmp0 (key, "official-repos") == 0) {
+		g_strfreev (priv->sources);
+		priv->sources = gs_plugin_provenance_get_sources (plugin);
+	}
+}
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_alloc_data (plugin, sizeof(GsPluginData));
+	priv->settings = g_settings_new ("org.gnome.software");
+	g_signal_connect (priv->settings, "changed",
+			  G_CALLBACK (gs_plugin_provenance_settings_changed_cb), plugin);
+	priv->sources = gs_plugin_provenance_get_sources (plugin);
+
+	/* after the package source is set */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "dummy");
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "packagekit-refine");
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "rpm-ostree");
+}
+
+void
+gs_plugin_destroy (GsPlugin *plugin)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	g_strfreev (priv->sources);
+	g_object_unref (priv->settings);
+}
+
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+	const gchar *origin;
+	gchar **sources;
+
+	/* not required */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_PROVENANCE) == 0)
+		return TRUE;
+	if (gs_app_has_quirk (app, GS_APP_QUIRK_PROVENANCE))
+		return TRUE;
+
+	/* nothing to search */
+	sources = priv->sources;
+	if (sources == NULL || sources[0] == NULL)
+		return TRUE;
+
+	/* simple case */
+	origin = gs_app_get_origin (app);
+	if (origin != NULL && gs_utils_strv_fnmatch (sources, origin)) {
+		gs_app_add_quirk (app, GS_APP_QUIRK_PROVENANCE);
+		return TRUE;
+	}
+
+	/* this only works for packages */
+	origin = gs_app_get_source_id_default (app);
+	if (origin == NULL)
+		return TRUE;
+	origin = g_strrstr (origin, ";");
+	if (origin == NULL)
+		return TRUE;
+	if (g_str_has_prefix (origin + 1, "installed:"))
+		origin += 10;
+	if (gs_utils_strv_fnmatch (sources, origin + 1)) {
+		gs_app_add_quirk (app, GS_APP_QUIRK_PROVENANCE);
+		return TRUE;
+	}
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	GsPluginData *priv = gs_plugin_get_data (plugin);
+
+	/* nothing to do here */
+	if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_PROVENANCE) == 0)
+		return TRUE;
+	/* nothing to search */
+	if (priv->sources == NULL || priv->sources[0] == NULL)
+		return TRUE;
+
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-plugin-rewrite-resource.c b/plugins/core/gs-plugin-rewrite-resource.c
new file mode 100644
index 0000000..2856a3e
--- /dev/null
+++ b/plugins/core/gs-plugin-rewrite-resource.c
@@ -0,0 +1,73 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2017 Richard Hughes <richard@hughsie.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <config.h>
+
+#include <glib/gi18n.h>
+#include <gnome-software.h>
+
+void
+gs_plugin_initialize (GsPlugin *plugin)
+{
+	/* let appstream add metadata first */
+	gs_plugin_add_rule (plugin, GS_PLUGIN_RULE_RUN_AFTER, "appstream");
+}
+
+static gboolean
+refine_app (GsPlugin             *plugin,
+	    GsApp                *app,
+	    GsPluginRefineFlags   flags,
+	    GCancellable         *cancellable,
+	    GError              **error)
+{
+	const gchar *keys[] = {
+		"GnomeSoftware::AppTile-css",
+		"GnomeSoftware::FeatureTile-css",
+		"GnomeSoftware::UpgradeBanner-css",
+		NULL };
+
+	/* rewrite URIs */
+	for (guint i = 0; keys[i] != NULL; i++) {
+		const gchar *css = gs_app_get_metadata_item (app, keys[i]);
+		if (css != NULL) {
+			g_autofree gchar *css_new = NULL;
+			g_autoptr(GsApp) app_dl = gs_app_new (gs_plugin_get_name (plugin));
+			gs_app_set_summary_missing (app_dl,
+						    /* TRANSLATORS: status text when downloading */
+						    _("Downloading featured images…"));
+			css_new = gs_plugin_download_rewrite_resource (plugin,
+								       app,
+								       css,
+								       cancellable,
+								       error);
+			if (css_new == NULL)
+				return FALSE;
+			if (g_strcmp0 (css, css_new) != 0) {
+				gs_app_set_metadata (app, keys[i], NULL);
+				gs_app_set_metadata (app, keys[i], css_new);
+			}
+		}
+	}
+	return TRUE;
+}
+
+gboolean
+gs_plugin_refine (GsPlugin             *plugin,
+		  GsAppList            *list,
+		  GsPluginRefineFlags   flags,
+		  GCancellable         *cancellable,
+		  GError              **error)
+{
+	for (guint i = 0; i < gs_app_list_length (list); i++) {
+		GsApp *app = gs_app_list_index (list, i);
+		if (!refine_app (plugin, app, flags, cancellable, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
diff --git a/plugins/core/gs-self-test.c b/plugins/core/gs-self-test.c
new file mode 100644
index 0000000..5aabbdf
--- /dev/null
+++ b/plugins/core/gs-self-test.c
@@ -0,0 +1,277 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ * vi:set noexpandtab tabstop=8 shiftwidth=8:
+ *
+ * Copyright (C) 2017 Joaquim Rocha <jrocha@endlessm.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include "config.h"
+
+#include <glib/gstdio.h>
+
+#include "gnome-software-private.h"
+
+#include "gs-appstream.h"
+#include "gs-test.h"
+
+static void
+gs_plugins_core_search_repo_name_func (GsPluginLoader *plugin_loader)
+{
+	GsApp *app;
+	g_autoptr(GError) error = NULL;
+	g_autoptr(GsApp) app_tmp = NULL;
+	g_autoptr(GsAppList) list = NULL;
+	g_autoptr(GsPluginJob) plugin_job = NULL;
+
+	/* drop all caches */
+	gs_utils_rmtree (g_getenv ("GS_SELF_TEST_CACHEDIR"), NULL);
+	gs_plugin_loader_setup_again (plugin_loader);
+
+	/* force this app to be installed */
+	app_tmp = gs_plugin_loader_app_create (plugin_loader, "*/*/yellow/desktop/arachne.desktop/*");
+	gs_app_set_state (app_tmp, AS_APP_STATE_INSTALLED);
+
+	/* get search result based on addon keyword */
+	plugin_job = gs_plugin_job_newv (GS_PLUGIN_ACTION_SEARCH,
+					 "search", "yellow",
+					 NULL);
+	gs_plugin_job_set_refine_flags (plugin_job, GS_PLUGIN_REFINE_FLAGS_REQUIRE_ICON);
+	list = gs_plugin_loader_job_process (plugin_loader, plugin_job, NULL, &error);
+	gs_test_flush_main_context ();
+	g_assert_no_error (error);
+	g_assert (list != NULL);
+
+	/* make sure there is one entry, the parent app */
+	g_assert_cmpint (gs_app_list_length (list), ==, 1);
+	app = gs_app_list_index (list, 0);
+	g_assert_cmpstr (gs_app_get_id (app), ==, "arachne.desktop");
+	g_assert_cmpint (gs_app_get_kind (app), ==, AS_APP_KIND_DESKTOP);
+}
+
+static void
+gs_plugins_core_os_release_func (GsPluginLoader *plugin_loader)
+{
+	gboolean ret;
+	g_autoptr(GsApp) app = NULL;
+	g_autoptr(GsApp) app3 = NULL;
+	g_autoptr(GsPluginJob) plugin_job = NULL;
+	g_autoptr(GError) error = NULL;
+
+	/* drop all caches */
+	gs_utils_rmtree (g_getenv ("GS_SELF_TEST_CACHEDIR"), NULL);
+	gs_plugin_loader_setup_again (plugin_loader);
+
+	/* refine system application */
+	app = gs_plugin_loader_get_system_app (plugin_loader);
+	plugin_job = gs_plugin_job_newv (GS_PLUGIN_ACTION_REFINE,
+					 "app", app,
+					 "refine-flags", GS_PLUGIN_REFINE_FLAGS_REQUIRE_URL |
+							 GS_PLUGIN_REFINE_FLAGS_REQUIRE_VERSION,
+					 NULL);
+	ret = gs_plugin_loader_job_action (plugin_loader, plugin_job, NULL, &error);
+	gs_test_flush_main_context ();
+	g_assert_no_error (error);
+	g_assert (ret);
+
+	/* make sure there is valid content */
+	g_assert_cmpstr (gs_app_get_id (app), ==, "org.fedoraproject.Fedora-25");
+	g_assert_cmpint (gs_app_get_kind (app), ==, AS_APP_KIND_OS_UPGRADE);
+	g_assert_cmpint (gs_app_get_state (app), ==, AS_APP_STATE_INSTALLED);
+	g_assert_cmpstr (gs_app_get_name (app), ==, "Fedora");
+	g_assert_cmpstr (gs_app_get_version (app), ==, "25");
+	g_assert_cmpstr (gs_app_get_url (app, AS_URL_KIND_HOMEPAGE), ==,
+			 "https://fedoraproject.org/");
+	g_assert_cmpstr (gs_app_get_metadata_item (app, "GnomeSoftware::CpeName"), ==,
+			 "cpe:/o:fedoraproject:fedora:25");
+
+	/* this comes from appstream */
+	g_assert_cmpstr (gs_app_get_summary (app), ==, "Fedora Workstation");
+
+	/* check we can get this by the old name too */
+	app3 = gs_plugin_loader_get_system_app (plugin_loader);
+	g_assert (app3 != NULL);
+	g_assert (app3 == app);
+}
+
+static void
+gs_plugins_core_generic_updates_func (GsPluginLoader *plugin_loader)
+{
+	gboolean ret;
+	GsApp *os_update;
+	GsAppList *related;
+	g_autoptr(GsPluginJob) plugin_job = NULL;
+	g_autoptr(GsPluginJob) plugin_job2 = NULL;
+	g_autoptr(GError) error = NULL;
+	g_autoptr(GsApp) app1 = NULL;
+	g_autoptr(GsApp) app2 = NULL;
+	g_autoptr(GsApp) app_wildcard = NULL;
+	g_autoptr(GsAppList) list = NULL;
+	g_autoptr(GsAppList) list_wildcard = NULL;
+
+	/* drop all caches */
+	gs_utils_rmtree (g_getenv ("GS_SELF_TEST_CACHEDIR"), NULL);
+	gs_plugin_loader_setup_again (plugin_loader);
+
+	/* create a list with generic apps */
+	list = gs_app_list_new ();
+	app1 = gs_app_new ("package1");
+	app2 = gs_app_new ("package2");
+	gs_app_set_kind (app1, AS_APP_KIND_GENERIC);
+	gs_app_set_kind (app2, AS_APP_KIND_GENERIC);
+	gs_app_set_bundle_kind (app1, AS_BUNDLE_KIND_PACKAGE);
+	gs_app_set_bundle_kind (app2, AS_BUNDLE_KIND_PACKAGE);
+	gs_app_set_scope (app1, AS_APP_SCOPE_SYSTEM);
+	gs_app_set_scope (app2, AS_APP_SCOPE_SYSTEM);
+	gs_app_set_state (app1, AS_APP_STATE_UPDATABLE);
+	gs_app_set_state (app2, AS_APP_STATE_UPDATABLE);
+	gs_app_add_source (app1, "package1");
+	gs_app_add_source (app2, "package2");
+	gs_app_list_add (list, app1);
+	gs_app_list_add (list, app2);
+
+	/* refine to make the generic-updates plugin merge them into a single OsUpdate item */
+	plugin_job = gs_plugin_job_newv (GS_PLUGIN_ACTION_REFINE,
+	                                 "list", list,
+	                                 "refine-flags", GS_PLUGIN_REFINE_FLAGS_REQUIRE_UPDATE_DETAILS,
+	                                 NULL);
+	ret = gs_plugin_loader_job_action (plugin_loader, plugin_job, NULL, &error);
+	gs_test_flush_main_context ();
+	g_assert_no_error (error);
+	g_assert (ret);
+
+	/* make sure there is one entry, the os update */
+	g_assert_cmpint (gs_app_list_length (list), ==, 1);
+	os_update = gs_app_list_index (list, 0);
+
+	/* make sure the os update is valid */
+	g_assert_cmpstr (gs_app_get_id (os_update), ==, "org.gnome.Software.OsUpdate");
+	g_assert_cmpint (gs_app_get_kind (os_update), ==, AS_APP_KIND_OS_UPDATE);
+	g_assert (gs_app_has_quirk (os_update, GS_APP_QUIRK_IS_PROXY));
+
+	/* must have two related apps, the ones we added earlier */
+	related = gs_app_get_related (os_update);
+	g_assert_cmpint (gs_app_list_length (related), ==, 2);
+
+	/* another test to make sure that we don't get an OsUpdate item created for wildcard apps */
+	list_wildcard = gs_app_list_new ();
+	app_wildcard = gs_app_new ("nosuchapp.desktop");
+	gs_app_add_quirk (app_wildcard, GS_APP_QUIRK_IS_WILDCARD);
+	gs_app_set_kind (app_wildcard, AS_APP_KIND_GENERIC);
+	gs_app_list_add (list_wildcard, app_wildcard);
+	plugin_job2 = gs_plugin_job_newv (GS_PLUGIN_ACTION_REFINE,
+	                                  "list", list_wildcard,
+	                                  "refine-flags", GS_PLUGIN_REFINE_FLAGS_REQUIRE_UPDATE_DETAILS,
+	                                  NULL);
+	ret = gs_plugin_loader_job_action (plugin_loader, plugin_job2, NULL, &error);
+	gs_test_flush_main_context ();
+	g_assert_no_error (error);
+	g_assert (ret);
+
+	/* no OsUpdate item created */
+	for (guint i = 0; i < gs_app_list_length (list_wildcard); i++) {
+		GsApp *app_tmp = gs_app_list_index (list_wildcard, i);
+		g_assert_cmpint (gs_app_get_kind (app_tmp), !=, AS_APP_KIND_OS_UPDATE);
+		g_assert (!gs_app_has_quirk (app_tmp, GS_APP_QUIRK_IS_PROXY));
+	}
+}
+
+int
+main (int argc, char **argv)
+{
+	g_autofree gchar *tmp_root = NULL;
+	gboolean ret;
+	int retval;
+	g_autofree gchar *os_release_filename = NULL;
+	g_autoptr(GError) error = NULL;
+	g_autoptr(GsPluginLoader) plugin_loader = NULL;
+	const gchar *xml;
+	const gchar *allowlist[] = {
+		"appstream",
+		"generic-updates",
+		"icons",
+		"os-release",
+		NULL
+	};
+
+	/* While we use %G_TEST_OPTION_ISOLATE_DIRS to create temporary directories
+	 * for each of the tests, we want to use the system MIME registry, assuming
+	 * that it exists and correctly has shared-mime-info installed. */
+#if GLIB_CHECK_VERSION(2, 60, 0)
+	g_content_type_set_mime_dirs (NULL);
+#endif
+
+	/* Similarly, add the system-wide icon theme path before it’s
+	 * overwritten by %G_TEST_OPTION_ISOLATE_DIRS. */
+	gs_test_expose_icon_theme_paths ();
+
+	g_test_init (&argc, &argv,
+#if GLIB_CHECK_VERSION(2, 60, 0)
+		     G_TEST_OPTION_ISOLATE_DIRS,
+#endif
+		     NULL);
+	g_setenv ("G_MESSAGES_DEBUG", "all", TRUE);
+
+	/* Use a common cache directory for all tests, since the appstream
+	 * plugin uses it and cannot be reinitialised for each test. */
+	tmp_root = g_dir_make_tmp ("gnome-software-core-test-XXXXXX", NULL);
+	g_assert (tmp_root != NULL);
+	g_setenv ("GS_SELF_TEST_CACHEDIR", tmp_root, TRUE);
+
+	os_release_filename = gs_test_get_filename (TESTDATADIR, "os-release");
+	g_assert (os_release_filename != NULL);
+	g_setenv ("GS_SELF_TEST_OS_RELEASE_FILENAME", os_release_filename, TRUE);
+
+	/* fake some data */
+	xml = "<?xml version=\"1.0\"?>\n"
+		"<components origin=\"yellow\" version=\"0.9\">\n"
+		"  <component type=\"desktop\">\n"
+		"    <id>arachne.desktop</id>\n"
+		"    <name>test</name>\n"
+		"    <summary>Test</summary>\n"
+		"    <icon type=\"stock\">system-file-manager</icon>\n"
+		"    <pkgname>arachne</pkgname>\n"
+		"  </component>\n"
+		"  <component type=\"os-upgrade\">\n"
+		"    <id>org.fedoraproject.Fedora-25</id>\n"
+		"    <name>Fedora</name>\n"
+		"    <summary>Fedora Workstation</summary>\n"
+		"    <pkgname>fedora-release</pkgname>\n"
+		"  </component>\n"
+		"  <info>\n"
+		"    <scope>user</scope>\n"
+		"  </info>\n"
+		"</components>\n";
+	g_setenv ("GS_SELF_TEST_APPSTREAM_XML", xml, TRUE);
+
+	/* only critical and error are fatal */
+	g_log_set_fatal_mask (NULL, G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL);
+
+	/* we can only load this once per process */
+	plugin_loader = gs_plugin_loader_new ();
+	gs_plugin_loader_add_location (plugin_loader, LOCALPLUGINDIR);
+	ret = gs_plugin_loader_setup (plugin_loader,
+				      (gchar**) allowlist,
+				      NULL,
+				      NULL,
+				      &error);
+	g_assert_no_error (error);
+	g_assert (ret);
+
+	/* plugin tests go here */
+	g_test_add_data_func ("/gnome-software/plugins/core/search-repo-name",
+			      plugin_loader,
+			      (GTestDataFunc) gs_plugins_core_search_repo_name_func);
+	g_test_add_data_func ("/gnome-software/plugins/core/os-release",
+			      plugin_loader,
+			      (GTestDataFunc) gs_plugins_core_os_release_func);
+	g_test_add_data_func ("/gnome-software/plugins/core/generic-updates",
+			      plugin_loader,
+			      (GTestDataFunc) gs_plugins_core_generic_updates_func);
+	retval = g_test_run ();
+
+	/* Clean up. */
+	gs_utils_rmtree (tmp_root, NULL);
+
+	return retval;
+}
diff --git a/plugins/core/meson.build b/plugins/core/meson.build
new file mode 100644
index 0000000..10c43f3
--- /dev/null
+++ b/plugins/core/meson.build
@@ -0,0 +1,250 @@
+cargs = ['-DG_LOG_DOMAIN="GsPlugin"']
+
+shared_module(
+  'gs_plugin_generic-updates',
+  sources : 'gs-plugin-generic-updates.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_key-colors',
+  sources : 'gs-plugin-key-colors.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_key-colors-metadata',
+  sources : 'gs-plugin-key-colors-metadata.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_provenance',
+  sources : 'gs-plugin-provenance.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_provenance-license',
+  sources : 'gs-plugin-provenance-license.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+
+shared_module(
+  'gs_plugin_icons',
+  sources : 'gs-plugin-icons.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_appstream',
+  sources : [
+    'gs-appstream.c',
+    'gs-plugin-appstream.c'
+  ],
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : [
+    plugin_libs,
+    libxmlb,
+  ],
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_desktop-categories',
+  sources : [
+    'gs-plugin-desktop-categories.c',
+    'gs-desktop-common.c',
+  ],
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_desktop-menu-path',
+  sources : [
+    'gs-plugin-desktop-menu-path.c',
+    'gs-desktop-common.c',
+  ],
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_hardcoded-blocklist',
+  sources : 'gs-plugin-hardcoded-blocklist.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+if get_option('hardcoded_popular')
+  shared_module(
+    'gs_plugin_hardcoded-popular',
+    sources : 'gs-plugin-hardcoded-popular.c',
+    include_directories : [
+      include_directories('../..'),
+      include_directories('../../lib'),
+    ],
+    install : true,
+    install_dir: plugin_dir,
+    c_args : cargs,
+    dependencies : plugin_libs,
+    link_with : [
+      libgnomesoftware
+    ]
+  )
+endif
+
+shared_module(
+  'gs_plugin_rewrite-resource',
+  sources : 'gs-plugin-rewrite-resource.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+shared_module(
+  'gs_plugin_os-release',
+  sources : 'gs-plugin-os-release.c',
+  include_directories : [
+    include_directories('../..'),
+    include_directories('../../lib'),
+  ],
+  install : true,
+  install_dir: plugin_dir,
+  c_args : cargs,
+  dependencies : plugin_libs,
+  link_with : [
+    libgnomesoftware
+  ]
+)
+
+if get_option('tests')
+  cargs += ['-DLOCALPLUGINDIR="' + meson.current_build_dir() + '"']
+  cargs += ['-DTESTDATADIR="' + join_paths(meson.current_source_dir(), 'tests') + '"']
+  e = executable(
+    'gs-self-test-core',
+    compiled_schemas,
+    sources : [
+      'gs-self-test.c',
+      'gs-appstream.c'
+    ],
+    include_directories : [
+      include_directories('../..'),
+      include_directories('../../lib'),
+    ],
+    dependencies : [
+      plugin_libs,
+      libxmlb,
+    ],
+    link_with : [
+      libgnomesoftware
+    ],
+    c_args : cargs,
+  )
+  test('gs-self-test-core', e, suite: ['plugins', 'core'], env: test_env)
+endif
diff --git a/plugins/core/tests/os-release b/plugins/core/tests/os-release
new file mode 120000
index 0000000..1efe264
--- /dev/null
+++ b/plugins/core/tests/os-release
@@ -0,0 +1 @@
+../../../data/tests/os-release
\ No newline at end of file