diff options
Diffstat (limited to '.gitlab-ci')
| -rw-r--r-- | .gitlab-ci/README.md | 22 | ||||
| -rwxr-xr-x | .gitlab-ci/cache-subprojects.sh | 11 | ||||
| -rw-r--r-- | .gitlab-ci/debian-stable.Dockerfile | 94 | ||||
| -rw-r--r-- | .gitlab-ci/fedora.Dockerfile | 96 | ||||
| -rwxr-xr-x | .gitlab-ci/meson-junit-report.py | 114 | ||||
| -rwxr-xr-x | .gitlab-ci/run-docker.sh | 132 | ||||
| -rwxr-xr-x | .gitlab-ci/run-tests.sh | 49 |
7 files changed, 518 insertions, 0 deletions
diff --git a/.gitlab-ci/README.md b/.gitlab-ci/README.md new file mode 100644 index 0000000..4eadf22 --- /dev/null +++ b/.gitlab-ci/README.md @@ -0,0 +1,22 @@ +# CI support stuff + +## Docker image + +GitLab CI jobs run in a Docker image, defined here. To update that image +(perhaps to install some more packages): + +1. Edit `.gitlab-ci/Dockerfile` with the changes you want +1. Run `.gitlab-ci/run-docker.sh build --base=debian-stable --base-version=1` to + build the new image (bump the version from the latest listed for that `base` + on https://gitlab.gnome.org/GNOME/gnome-software/container_registry) +1. Run `.gitlab-ci/run-docker.sh push --base=debian-stable --base-version=1` to + upload the new image to the GNOME GitLab Docker registry + * If this is the first time you're doing this, you'll need to log into the + registry + * If you use 2-factor authentication on your GNOME GitLab account, you'll + need to [create a personal access token][pat] and use that rather than + your normal password +1. Edit `.gitlab-ci.yml` (in the root of this repository) to use your new + image + +[pat]: https://gitlab.gnome.org/-/profile/personal_access_tokens diff --git a/.gitlab-ci/cache-subprojects.sh b/.gitlab-ci/cache-subprojects.sh new file mode 100755 index 0000000..fb0337c --- /dev/null +++ b/.gitlab-ci/cache-subprojects.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +git clone --depth 1 --no-tags https://gitlab.gnome.org/GNOME/gnome-software.git +meson subprojects download --sourcedir gnome-software +rm gnome-software/subprojects/*.wrap +mv gnome-software/subprojects/ . +# allow updating this one without a docker rebuild +rm -rf subprojects/gnome-pwa-list +rm -rf gnome-software diff --git a/.gitlab-ci/debian-stable.Dockerfile b/.gitlab-ci/debian-stable.Dockerfile new file mode 100644 index 0000000..ab31fbb --- /dev/null +++ b/.gitlab-ci/debian-stable.Dockerfile @@ -0,0 +1,94 @@ +FROM debian:bookworm + +RUN apt-get update -qq && apt-get install --no-install-recommends -qq -y \ + appstream \ + clang \ + clang-tools \ + dbus \ + desktop-file-utils \ + docbook-xsl \ + gcc \ + g++ \ + gettext \ + git \ + gnome-pkg-tools \ + gobject-introspection \ + gperf \ + gsettings-desktop-schemas-dev \ + gtk-doc-tools \ + itstool \ + lcov \ + libaccountsservice-dev \ + libappstream-dev \ + libcairo2-dev \ + libcairo-gobject2 \ + libcurl4-gnutls-dev \ + libepoxy-dev \ + libflatpak-dev \ + libfontconfig-dev \ + libfwupd-dev \ + libgdk-pixbuf-2.0-dev \ + libgirepository1.0-dev \ + libglib2.0-dev \ + libglib-testing-0-dev \ + libgoa-1.0-dev \ + libgraphene-1.0-dev \ + libgstreamer-plugins-bad1.0-dev \ + libgudev-1.0-dev \ + libjpeg62-turbo-dev \ + libjson-glib-dev \ + liblmdb-dev \ + libmalcontent-0-dev \ + libpackagekit-glib2-dev \ + libpam0g-dev \ + libpango1.0-dev \ + libpolkit-gobject-1-dev \ + libsoup2.4-dev \ + libstemmer-dev \ + libxcursor-dev \ + libxdamage-dev \ + libxext-dev \ + libxfixes-dev \ + libxi-dev \ + libxinerama-dev \ + libxkbcommon-dev \ + libxmlb-dev \ + libxml2-utils \ + libxrandr-dev \ + libyaml-dev \ + ninja-build \ + packagekit \ + pkg-config \ + policykit-1 \ + python3 \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + sassc \ + shared-mime-info \ + sudo \ + sysprof \ + unzip \ + valgrind \ + wayland-protocols \ + wget \ + xsltproc \ + xz-utils \ + && rm -rf /usr/share/doc/* /usr/share/man/* + +RUN pip3 install meson==0.60.1 + +# Enable passwordless sudo for sudo users +RUN sed -i -e '/%sudo/s/ALL$/NOPASSWD: ALL/' /etc/sudoers + +ARG HOST_USER_ID=5555 +ENV HOST_USER_ID ${HOST_USER_ID} +RUN useradd -u $HOST_USER_ID -G sudo -ms /bin/bash user + +USER user +WORKDIR /home/user + +COPY cache-subprojects.sh . +RUN ./cache-subprojects.sh + +ENV LANG=C.UTF-8 LANGUAGE=C.UTF-8 LC_ALL=C.UTF-8 diff --git a/.gitlab-ci/fedora.Dockerfile b/.gitlab-ci/fedora.Dockerfile new file mode 100644 index 0000000..a3bc5f1 --- /dev/null +++ b/.gitlab-ci/fedora.Dockerfile @@ -0,0 +1,96 @@ +FROM fedora:35 + +RUN dnf -y install \ + appstream \ + accountsservice-devel \ + cairo-devel \ + cairo-gobject-devel \ + clang \ + clang-analyzer \ + dbus-daemon \ + dbus-devel \ + desktop-file-utils \ + docbook-style-xsl \ + flatpak-devel \ + fwupd-devel \ + gcc \ + gdk-pixbuf2-devel \ + gettext \ + git \ + glib2-devel \ + gobject-introspection \ + gobject-introspection-devel \ + gperf \ + graphene-devel \ + gsettings-desktop-schemas-devel \ + gstreamer1-plugins-bad-free-devel \ + gtk-doc \ + json-glib-devel \ + itstool \ + lcov \ + lmdb-devel \ + appstream-devel \ + libcurl-devel \ + libdnf-devel \ + libepoxy-devel \ + libglib-testing-devel \ + libgudev-devel \ + libjpeg-turbo-devel \ + liboauth-devel \ + libsecret-devel \ + libsoup-devel \ + libstemmer-devel \ + libXcursor-devel \ + libXdamage-devel \ + libXext-devel \ + libXfixes-devel \ + libXi-devel \ + libXinerama-devel \ + libxkbcommon-devel \ + libxmlb-devel \ + libXrandr-devel \ + libxslt \ + libyaml-devel \ + malcontent-devel \ + NetworkManager-libnm-devel \ + ninja-build \ + ostree-devel \ + PackageKit \ + PackageKit-glib-devel \ + pam-devel \ + pango-devel \ + pcre-devel \ + polkit-devel \ + python3 \ + python3-pip \ + python3-wheel \ + rpm-devel \ + rpm-ostree-devel \ + sassc \ + shared-mime-info \ + snapd-glib-devel \ + sysprof-capture-devel \ + unzip \ + valgrind \ + wayland-protocols-devel \ + wget \ + xz \ + zlib-devel \ + && dnf clean all + +RUN pip3 install meson==0.60.1 + +# Enable sudo for wheel users +RUN sed -i -e 's/# %wheel/%wheel/' -e '0,/%wheel/{s/%wheel/# %wheel/}' /etc/sudoers + +ARG HOST_USER_ID=5555 +ENV HOST_USER_ID ${HOST_USER_ID} +RUN useradd -u $HOST_USER_ID -G wheel -ms /bin/bash user + +USER user +WORKDIR /home/user + +COPY cache-subprojects.sh . +RUN ./cache-subprojects.sh + +ENV LANG C.UTF-8 diff --git a/.gitlab-ci/meson-junit-report.py b/.gitlab-ci/meson-junit-report.py new file mode 100755 index 0000000..8bb7667 --- /dev/null +++ b/.gitlab-ci/meson-junit-report.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# Turns a Meson testlog.json file into a JUnit XML report +# +# Copyright 2019 GNOME Foundation +# +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# Original author: Emmanuele Bassi + +import argparse +import datetime +import json +import os +import sys +import xml.etree.ElementTree as ET + +aparser = argparse.ArgumentParser(description='Turns a Meson test log into a JUnit report') +aparser.add_argument('--project-name', metavar='NAME', + help='The project name', + default='unknown') +aparser.add_argument('--job-id', metavar='ID', + help='The job ID for the report', + default='Unknown') +aparser.add_argument('--branch', metavar='NAME', + help='Branch of the project being tested', + default='main') +aparser.add_argument('--output', metavar='FILE', + help='The output file, stdout by default', + type=argparse.FileType('w', encoding='UTF-8'), + default=sys.stdout) +aparser.add_argument('infile', metavar='FILE', + help='The input testlog.json, stdin by default', + type=argparse.FileType('r', encoding='UTF-8'), + default=sys.stdin) + +args = aparser.parse_args() + +outfile = args.output + +testsuites = ET.Element('testsuites') +testsuites.set('id', '{}/{}'.format(args.job_id, args.branch)) +testsuites.set('package', args.project_name) +testsuites.set('timestamp', datetime.datetime.utcnow().isoformat()) + +suites = {} +for line in args.infile: + data = json.loads(line) + (full_suite, unit_name) = data['name'].split(' / ') + try: + (project_name, suite_name) = full_suite.split(':') + except ValueError: + (project_name, suite_name) = (args.project_name, full_suite) + + duration = data['duration'] + return_code = data['returncode'] + log = data['stdout'] + log_stderr = data.get('stderr', '') + + unit = { + 'suite': suite_name, + 'name': unit_name, + 'duration': duration, + 'returncode': return_code, + 'stdout': log, + 'stderr': log_stderr, + } + + units = suites.setdefault(suite_name, []) + units.append(unit) + +for name, units in suites.items(): + print('Processing suite {} (units: {})'.format(name, len(units))) + + def if_failed(unit): + if unit['returncode'] != 0: + return True + return False + + def if_succeded(unit): + if unit['returncode'] == 0: + return True + return False + + successes = list(filter(if_succeded, units)) + failures = list(filter(if_failed, units)) + print(' - {}: {} pass, {} fail'.format(name, len(successes), len(failures))) + + testsuite = ET.SubElement(testsuites, 'testsuite') + testsuite.set('name', '{}/{}'.format(args.project_name, name)) + testsuite.set('tests', str(len(units))) + testsuite.set('errors', str(len(failures))) + testsuite.set('failures', str(len(failures))) + + for unit in successes: + testcase = ET.SubElement(testsuite, 'testcase') + testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite'])) + testcase.set('name', unit['name']) + testcase.set('time', str(unit['duration'])) + + for unit in failures: + testcase = ET.SubElement(testsuite, 'testcase') + testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite'])) + testcase.set('name', unit['name']) + testcase.set('time', str(unit['duration'])) + + failure = ET.SubElement(testcase, 'failure') + failure.set('classname', '{}/{}'.format(args.project_name, unit['suite'])) + failure.set('name', unit['name']) + failure.set('type', 'error') + failure.text = unit['stdout'] + '\n' + unit['stderr'] + +output = ET.tostring(testsuites, encoding='unicode') +outfile.write(output) diff --git a/.gitlab-ci/run-docker.sh b/.gitlab-ci/run-docker.sh new file mode 100755 index 0000000..030cade --- /dev/null +++ b/.gitlab-ci/run-docker.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +read_arg() { + # $1 = arg name + # $2 = arg value + # $3 = arg parameter + local rematch='^[^=]*=(.*)$' + if [[ $2 =~ $rematch ]]; then + read -r "$1" <<< "${BASH_REMATCH[1]}" + else + read -r "$1" <<< "$3" + # There is no way to shift our callers args, so + # return 1 to indicate they should do it instead. + return 1 + fi +} + +SUDO_CMD="sudo" +if docker -v |& grep -q podman; then + # Using podman + SUDO_CMD="" + # Docker is actually implemented by podman, and its OCI output + # is incompatible with some of the dockerd instances on GitLab + # CI runners. + export BUILDAH_FORMAT=docker +fi + +set -e + +base="" +base_version="" +build=0 +run=0 +push=0 +list=0 +print_help=0 +no_login=0 + +while (($# > 0)); do + case "${1%%=*}" in + build) build=1;; + run) run=1;; + push) push=1;; + list) list=1;; + help) print_help=1;; + --base|-b) read_arg base "$@" || shift;; + --base-version) read_arg base_version "$@" || shift;; + --no-login) no_login=1;; + *) echo -e "\e[1;31mERROR\e[0m: Unknown option '$1'"; exit 1;; + esac + shift +done + +if [ $print_help == 1 ]; then + echo "$0 - Build and run Docker images" + echo "" + echo "Usage: $0 <command> [options] [basename]" + echo "" + echo "Available commands" + echo "" + echo " build --base=<BASENAME> - Build Docker image <BASENAME>.Dockerfile" + echo " run --base=<BASENAME> - Run Docker image <BASENAME>" + echo " push --base=<BASENAME> - Push Docker image <BASENAME> to the registry" + echo " list - List available images" + echo " help - This help message" + echo "" + exit 0 +fi + +cd "$(dirname "$0")" + +if [ $list == 1 ]; then + echo "Available Docker images:" + for f in *.Dockerfile; do + filename=$( basename -- "$f" ) + basename="${filename%.*}" + + echo -e " \e[1;39m$basename\e[0m" + done + exit 0 +fi + +# All commands after this require --base to be set +if [ -z "${base}" ]; then + echo "Usage: $0 <command>" + exit 1 +fi + +if [ ! -f "$base.Dockerfile" ]; then + echo -e "\e[1;31mERROR\e[0m: Dockerfile for '$base' not found" + exit 1 +fi + +if [ -z "${base_version}" ]; then + base_version="latest" +else + base_version="v$base_version" +fi + +TAG="registry.gitlab.gnome.org/gnome/gnome-software/${base}:${base_version}" + +if [ $build == 1 ]; then + echo -e "\e[1;32mBUILDING\e[0m: ${base} as ${TAG}" + $SUDO_CMD docker build \ + --build-arg HOST_USER_ID="$UID" \ + --tag "${TAG}" \ + --file "${base}.Dockerfile" . + exit $? +fi + +if [ $push == 1 ]; then + echo -e "\e[1;32mPUSHING\e[0m: ${base} as ${TAG}" + + if [ $no_login == 0 ]; then + $SUDO_CMD docker login registry.gitlab.gnome.org + fi + + $SUDO_CMD docker push $TAG + exit $? +fi + +if [ $run == 1 ]; then + echo -e "\e[1;32mRUNNING\e[0m: ${base} as ${TAG}" + $SUDO_CMD docker run \ + --rm \ + --volume "$(pwd)/..:/home/user/app" \ + --workdir "/home/user/app" \ + --tty \ + --interactive "${TAG}" \ + bash + exit $? +fi diff --git a/.gitlab-ci/run-tests.sh b/.gitlab-ci/run-tests.sh new file mode 100755 index 0000000..38e7cb1 --- /dev/null +++ b/.gitlab-ci/run-tests.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set +e + +case "$1" in + --log-file) + log_file="$2" + shift + shift + ;; + *) + log_file="_build/meson-logs/testlog.json" +esac + +# Remove Flatpak triggers, to speed up the tests (specifically on Fedora, +# they could even break the tests by a deadlock or something like that). +if [ -d /usr/share/flatpak/triggers ]; then + sudo rm -rf /usr/share/flatpak/triggers +fi + +# FIXME: The tests should be isolated and use mock services so they do not +# require a functioning system bus. This will have to do for now though. +sudo mkdir -p /run/dbus +sudo mkdir -p /var +sudo ln -s /var/run /run +sudo dbus-daemon --system --fork +#sudo /usr/lib/polkit-1/polkitd --no-debug & +#sudo /usr/libexec/fwupd/fwupd --verbose & + +# FIXME: Running the flatpak tests as root means the system helper doesn’t +# need to be used, which makes them run a lot faster. +sudo \ +dbus-run-session -- \ +meson test \ + -C _build \ + --timeout-multiplier ${MESON_TEST_TIMEOUT_MULTIPLIER} \ + --no-suite flaky \ + --print-errorlogs \ + "$@" + +exit_code=$? + +python3 .gitlab-ci/meson-junit-report.py \ + --project-name=gnome-software \ + --job-id "${CI_JOB_NAME}" \ + --output "_build/${CI_JOB_NAME}-report.xml" \ + "${log_file}" + +exit $exit_code |