From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- taskcluster/docker/snap-coreXX-build/Dockerfile | 80 ++ .../docker/snap-coreXX-build/install-snap.sh | 24 + taskcluster/docker/snap-coreXX-build/parse.py | 39 + taskcluster/docker/snap-coreXX-build/run.sh | 114 +++ .../docker/snap-coreXX-build/snap-tests/README.md | 24 + .../snap-coreXX-build/snap-tests/basic_tests.py | 307 ++++++++ .../snap-tests/basic_tests/expectations.json.in | 14 + .../snap-coreXX-build/snap-tests/qa_tests.py | 860 +++++++++++++++++++++ .../snap-tests/qa_tests/qa_expectations.json | 98 +++ .../snap-tests/qa_tests/test_custom_fonts_ref.png | Bin 0 -> 51060 bytes .../snap-tests/qa_tests/test_pdf_download_base.png | Bin 0 -> 127756 bytes .../qa_tests/test_pdf_navigation_base.png | Bin 0 -> 167492 bytes .../test_pdf_navigation_documentProperties.png | Bin 0 -> 177679 bytes .../qa_tests/test_pdf_navigation_down.png | Bin 0 -> 242142 bytes .../qa_tests/test_pdf_navigation_end.png | Bin 0 -> 135099 bytes .../qa_tests/test_pdf_navigation_firstPage.png | Bin 0 -> 194764 bytes .../qa_tests/test_pdf_navigation_hand_tool.png | Bin 0 -> 197844 bytes .../qa_tests/test_pdf_navigation_home.png | Bin 0 -> 194764 bytes .../qa_tests/test_pdf_navigation_lastPage.png | Bin 0 -> 135099 bytes .../qa_tests/test_pdf_navigation_left.png | Bin 0 -> 194878 bytes .../qa_tests/test_pdf_navigation_next.png | Bin 0 -> 247191 bytes .../qa_tests/test_pdf_navigation_pageRotateCcw.png | Bin 0 -> 194810 bytes .../qa_tests/test_pdf_navigation_pageRotateCw.png | Bin 0 -> 276055 bytes .../qa_tests/test_pdf_navigation_pagedown.png | Bin 0 -> 216369 bytes .../qa_tests/test_pdf_navigation_pageup.png | Bin 0 -> 171383 bytes .../qa_tests/test_pdf_navigation_previous.png | Bin 0 -> 224991 bytes .../qa_tests/test_pdf_navigation_right.png | Bin 0 -> 247234 bytes .../qa_tests/test_pdf_navigation_select_text.png | Bin 0 -> 195176 bytes .../snap-tests/qa_tests/test_pdf_navigation_up.png | Bin 0 -> 224891 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_100p.png | Bin 0 -> 194764 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_150p.png | Bin 0 -> 166658 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_400p.png | Bin 0 -> 72156 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_50p.png | Bin 0 -> 150887 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_75p.png | Bin 0 -> 155313 bytes .../qa_tests/test_pdf_zoom_p1_actual.png | Bin 0 -> 195201 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_fit.png | Bin 0 -> 162736 bytes .../snap-tests/qa_tests/test_pdf_zoom_p1_width.png | Bin 0 -> 188823 bytes .../snap-coreXX-build/snap-tests/requirements.txt | 22 + .../docker/snap-coreXX-build/snap-tests/tests.sh | 68 ++ .../snap-tests/update-references.sh | 22 + 40 files changed, 1672 insertions(+) create mode 100644 taskcluster/docker/snap-coreXX-build/Dockerfile create mode 100755 taskcluster/docker/snap-coreXX-build/install-snap.sh create mode 100644 taskcluster/docker/snap-coreXX-build/parse.py create mode 100755 taskcluster/docker/snap-coreXX-build/run.sh create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/README.md create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/qa_expectations.json create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_custom_fonts_ref.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_download_base.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_base.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_documentProperties.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_down.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_end.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_firstPage.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_hand_tool.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_home.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_lastPage.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_left.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_next.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCcw.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageRotateCw.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pagedown.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_pageup.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_previous.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_right.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_select_text.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_navigation_up.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_100p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_150p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_400p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_50p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_75p.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_actual.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_fit.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests/test_pdf_zoom_p1_width.png create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/requirements.txt create mode 100755 taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh create mode 100755 taskcluster/docker/snap-coreXX-build/snap-tests/update-references.sh (limited to 'taskcluster/docker/snap-coreXX-build') diff --git a/taskcluster/docker/snap-coreXX-build/Dockerfile b/taskcluster/docker/snap-coreXX-build/Dockerfile new file mode 100644 index 0000000000..29a343f6ea --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/Dockerfile @@ -0,0 +1,80 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Gabriele Svelto + +RUN mkdir -p /builds +RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker +WORKDIR /builds/worker + +# We need to declare all potentially cache volumes as caches. Also, +# making high I/O paths volumes increase I/O throughput because of +# AUFS slowness. +VOLUME /builds/worker/checkouts + +RUN apt-get update && \ + apt-get install -y \ + build-essential \ + curl \ + libavcodec58 \ + libavutil56 \ + jq \ + patch \ + patchelf \ + python3-dev \ + python3-yaml \ + squashfs-tools \ + tar \ + unzip \ + uuid \ + wget \ + zip + +ARG SNAP_BASE + +COPY install-snap.sh /usr/local/bin/ + +RUN install-snap.sh core + +RUN install-snap.sh core20 + +# Snapcraft snap depends on core20 for python3.8 even when we target core22 +RUN install-snap.sh snapcraft + +RUN install-snap.sh core22 + +ARG SNAP_LIST + +RUN for snap in $SNAP_LIST; do install-snap.sh "${snap}"; done + +# Create a snapcraft runner +RUN mkdir -p /snap/bin +RUN echo "#!/bin/sh" > /snap/bin/snapcraft +RUN snap_version="$(awk '/^version:/{print $2}' /snap/snapcraft/current/meta/snap.yaml)" && echo "export SNAP_VERSION=\"$snap_version\"\nexport PATH=\$SNAP/bin/:\$SNAP/libexec/snapcraft/:\$PATH" >> /snap/bin/snapcraft +RUN echo 'exec "$SNAP/bin/snapcraft" "$@"' >> /snap/bin/snapcraft +RUN chmod +x /snap/bin/snapcraft + +# Generate locale +RUN apt update && apt dist-upgrade --yes && apt install --yes sudo snapd locales && locale-gen en_US.UTF-8 && apt-get autoremove -y + +# Set the proper environment +ENV LANG="en_US.UTF-8" +ENV LANGUAGE="en_US:en" +ENV LC_ALL="en_US.UTF-8" +ENV PATH="/snap/bin:$PATH" +ENV SNAP="/snap/snapcraft/current" +ENV SNAP_NAME="snapcraft" +ENV SNAP_ARCH="amd64" + +# Snapcraft pull will need to sudo apt-get update ... +RUN echo "worker ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/worker && \ + chmod 0440 /etc/sudoers.d/worker + +COPY run.sh /builds/worker/run.sh + +COPY parse.py /builds/worker/parse.py + +COPY patches /builds/worker/patches/ + +RUN chown -R worker:worker /builds/worker + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/snap-coreXX-build/install-snap.sh b/taskcluster/docker/snap-coreXX-build/install-snap.sh new file mode 100755 index 0000000000..66f64d088d --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/install-snap.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -ex + +SNAP_TO_INSTALL=$1 + +if [ -z "${SNAP_TO_INSTALL}" ]; then + echo "Please give a snap name" + exit 1 +fi + +# Grab the requested snap from the stable channel and unpack it in the proper +# place (the 'Snap-CDN: none' header allows building in restricted network +# environments such as Launchpad builders) + +# shellcheck disable=SC2046 +curl -L \ + -H 'Snap-CDN: none' \ + $(curl -H 'X-Ubuntu-Series: 16' "https://api.snapcraft.io/api/v1/snaps/details/${SNAP_TO_INSTALL}?channel=stable" | jq '.download_url' -r) \ + --output "${SNAP_TO_INSTALL}.snap" + +mkdir -p "/snap/${SNAP_TO_INSTALL}" + +unsquashfs -d "/snap/${SNAP_TO_INSTALL}/current" "${SNAP_TO_INSTALL}.snap" diff --git a/taskcluster/docker/snap-coreXX-build/parse.py b/taskcluster/docker/snap-coreXX-build/parse.py new file mode 100644 index 0000000000..82e0e2f4a6 --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/parse.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import sys + +import yaml + + +def has_pkg_section(p, section): + has_section = section in p.keys() + if has_section: + for pkg in p[section]: + yield pkg + + +def iter_pkgs(part, all_pkgs): + for pkg in has_pkg_section(part, "build-packages"): + if pkg not in all_pkgs: + all_pkgs.append(pkg) + for pkg in has_pkg_section(part, "stage-packages"): + if pkg not in all_pkgs: + all_pkgs.append(pkg) + + +def parse(yaml_file): + all_pkgs = [] + with open(yaml_file, "r") as inp: + snap = yaml.safe_load(inp) + parts = snap["parts"] + for p in parts: + iter_pkgs(parts[p], all_pkgs) + return " ".join(all_pkgs) + + +if __name__ == "__main__": + print(parse(sys.argv[1])) diff --git a/taskcluster/docker/snap-coreXX-build/run.sh b/taskcluster/docker/snap-coreXX-build/run.sh new file mode 100755 index 0000000000..5b3efbb756 --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/run.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +set -ex + +mkdir -p /builds/worker/artifacts/ +mkdir -p /builds/worker/.local/state/snapcraft/ +ln -s /builds/worker/artifacts /builds/worker/.local/state/snapcraft/log + +BRANCH=$1 +DEBUG=${2:-0} + +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 +export SNAP_ARCH=amd64 +export SNAPCRAFT_BUILD_INFO=1 + +export PATH=$PATH:$HOME/.local/bin/ +unset MOZ_AUTOMATION + +MOZCONFIG=mozconfig.in + +USE_SNAP_FROM_STORE=${USE_SNAP_FROM_STORE:-0} + +TRY=0 +if [ "${BRANCH}" = "try" ]; then + BRANCH=nightly + TRY=1 +fi + +if [ "${USE_SNAP_FROM_STORE}" = "0" ]; then + # ESR currently still has a hard dependency against zstandard==0.17.0 so + # install this specific version here + if [ "${BRANCH}" = "esr" ]; then + sudo apt-get remove -y python3-zstandard && sudo apt-get install -y python3-pip && sudo pip3 install --no-input zstandard==0.17.0 + MOZCONFIG=mozconfig + fi + + # Stable and beta runs out of file descriptors during link with gold + ulimit -n 65536 + + git clone --single-branch --depth 1 --branch "${BRANCH}" https://github.com/canonical/firefox-snap/ + cd firefox-snap/ + + if [ "${TRY}" = "1" ]; then + # Symlink so that we can directly re-use Gecko mercurial checkout + ln -s /builds/worker/checkouts/gecko gecko + fi + + # Force an update to avoid the case of a stale docker image and repos updated + # after + sudo apt-get update + + # shellcheck disable=SC2046 + sudo apt-get install -y $(/usr/bin/python3 /builds/worker/parse.py snapcraft.yaml) + + # CRAFT_PARTS_PACKAGE_REFRESH required to avoid snapcraft running apt-get update + # especially for stage-packages + if [ -d "/builds/worker/patches/${BRANCH}/" ]; then + for p in /builds/worker/patches/"${BRANCH}"/*.patch; do + patch -p1 < "$p" + done; + fi + + if [ "${TRY}" = "1" ]; then + # don't remove hg source, and don't force changeset so we get correct stamp + # still force repo because the try clone is from mozilla-unified but the + # generated link does not work + sed -ri 's|rm -rf .hg||g' snapcraft.yaml + # shellcheck disable=SC2016 + sed -ri 's|MOZ_SOURCE_REPO=\$\{REPO\}|MOZ_SOURCE_REPO=${GECKO_HEAD_REPOSITORY}|g' snapcraft.yaml + # shellcheck disable=SC2016 + sed -ri 's|MOZ_SOURCE_CHANGESET=\$\{REVISION\}|MOZ_SOURCE_CHANGESET=${GECKO_HEAD_REV}|g' snapcraft.yaml + # shellcheck disable=SC2016 + sed -ri 's|hg clone --stream \$REPO -u \$REVISION|cp -r \$SNAPCRAFT_PROJECT_DIR/gecko/. |g' snapcraft.yaml + fi + + if [ "${DEBUG}" = "1" ]; then + { + echo "ac_add_options --enable-debug" + echo "ac_add_options --disable-install-strip" + } >> ${MOZCONFIG} + echo "MOZ_DEBUG=1" >> ${MOZCONFIG} + + # No PGO on debug builds + sed -ri 's/ac_add_options --enable-linker=gold//g' snapcraft.yaml + sed -ri 's/ac_add_options --enable-lto=cross//g' snapcraft.yaml + sed -ri 's/ac_add_options MOZ_PGO=1//g' snapcraft.yaml + fi + + SNAPCRAFT_BUILD_ENVIRONMENT_MEMORY=64G \ + SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \ + CRAFT_PARTS_PACKAGE_REFRESH=0 \ + snapcraft --destructive-mode --verbose +else + mkdir from-snap-store && cd from-snap-store + + CHANNEL="${BRANCH}" + if [ "${CHANNEL}" = "try" ] || [ "${CHANNEL}" = "nightly" ]; then + CHANNEL=edge + fi; + + snap download --channel="${CHANNEL}" firefox + SNAP_DEBUG_NAME=$(find . -maxdepth 1 -type f -name "firefox*.snap" | sed -e 's/\.snap$/.debug/g') + touch "${SNAP_DEBUG_NAME}" +fi + +cp ./*.snap ./*.debug /builds/worker/artifacts/ + +# Those are for fetches usage by the test task +cp ./*.snap /builds/worker/artifacts/firefox.snap +cp ./*.debug /builds/worker/artifacts/firefox.debug + +# Those are for running snap-upstream-test +cd /builds/worker/checkouts/gecko/taskcluster/docker/snap-coreXX-build/snap-tests/ && zip -r9 /builds/worker/artifacts/snap-tests.zip ./* diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/README.md b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md new file mode 100644 index 0000000000..73f78c72bc --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md @@ -0,0 +1,24 @@ +Debugging tests +================ + +You can use the `TEST_FILTER` environment variable, e.g., `TEST_FILTER=xxx` +will filter test named `test_xxx`. + +Setting `TEST_GECKODRIVER_TRACE` to any value will make Selenium dump a trace +log for debugging. + +You can control running headless or not with `TEST_NO_HEADLESS`. Currently, +the copy/paste image test required NOT to run headless. + +More useful for local repro, you can set `TEST_NO_QUIT` if you need to keep +inspecting the browser at the end of a test. + +Data URL containing the diff screenshot will be dumped to stdout/stderr when +`TEST_DUMP_DIFF` is set in the environment. + +Updating reference screenshots +============================== + - `./mach try fuzzy --push-to-lando --full --env TEST_COLLECT_REFERENCE=1 -q "'snap-upstream-test"` + - note the successfull task id you want to source + - you need curl and jq installed + - ./update-references.sh diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py new file mode 100644 index 0000000000..fb8841817b --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + + +import base64 +import io +import json +import os +import sys +import time +import traceback + +from mozlog import formatters, handlers, structuredlog +from PIL import Image, ImageChops +from selenium import webdriver +from selenium.common.exceptions import TimeoutException +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.webdriver.firefox.service import Service +from selenium.webdriver.remote.webelement import WebElement +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + + +class SnapTestsBase: + def __init__(self, exp): + driver_service = Service( + executable_path=r"/snap/firefox/current/usr/lib/firefox/geckodriver", + log_output=os.path.join( + os.environ.get("ARTIFACT_DIR", ""), "geckodriver.log" + ), + ) + options = Options() + if "TEST_GECKODRIVER_TRACE" in os.environ.keys(): + options.log.level = "trace" + options.binary_location = r"/snap/firefox/current/usr/lib/firefox/firefox" + if not "TEST_NO_HEADLESS" in os.environ.keys(): + options.add_argument("--headless") + if "MOZ_AUTOMATION" in os.environ.keys(): + os.environ["MOZ_LOG_FILE"] = os.path.join( + os.environ.get("ARTIFACT_DIR"), "gecko.log" + ) + self._driver = webdriver.Firefox(service=driver_service, options=options) + + self._logger = structuredlog.StructuredLogger(self.__class__.__name__) + self._logger.add_handler( + handlers.StreamHandler(sys.stdout, formatters.TbplFormatter()) + ) + + test_filter = "test_{}".format(os.environ.get("TEST_FILTER", "")) + object_methods = [ + method_name + for method_name in dir(self) + if callable(getattr(self, method_name)) + and method_name.startswith(test_filter) + ] + + self._logger.suite_start(object_methods) + + assert self._dir is not None + + self._wait = WebDriverWait(self._driver, self.get_timeout()) + self._longwait = WebDriverWait(self._driver, 60) + + with open(exp, "r") as j: + self._expectations = json.load(j) + + rv = False + try: + first_tab = self._driver.window_handles[0] + for m in object_methods: + tabs_before = set(self._driver.window_handles) + self._driver.switch_to.window(first_tab) + self._logger.test_start(m) + rv = getattr(self, m)(self._expectations[m]) + self._driver.switch_to.parent_frame() + if rv: + self._logger.test_end(m, status="OK") + else: + self._logger.test_end(m, status="FAIL") + tabs_after = set(self._driver.window_handles) + tabs_opened = tabs_after - tabs_before + self._logger.info("opened {} tabs".format(len(tabs_opened))) + for tab in tabs_opened: + self._driver.switch_to.window(tab) + self._driver.close() + self._wait.until(EC.number_of_windows_to_be(len(tabs_before))) + except Exception as ex: + rv = False + test_status = "ERROR" + if isinstance(ex, AssertionError): + test_status = "FAIL" + elif isinstance(ex, TimeoutException): + test_status = "TIMEOUT" + + test_message = repr(ex) + self.save_screenshot("screenshot_{}.png".format(test_status.lower())) + self._driver.switch_to.parent_frame() + self.save_screenshot("screenshot_{}_parent.png".format(test_status.lower())) + self._logger.test_end(m, status=test_status, message=test_message) + traceback.print_exc() + finally: + self._driver.switch_to.window(first_tab) + + if not "TEST_NO_QUIT" in os.environ.keys(): + self._driver.quit() + + self._logger.info("Exiting with {}".format(rv)) + self._logger.suite_end() + sys.exit(0 if rv is True else 1) + + def get_screenshot_destination(self, name): + final_name = name + if "MOZ_AUTOMATION" in os.environ.keys(): + final_name = os.path.join(os.environ.get("ARTIFACT_DIR"), name) + return final_name + + def save_screenshot(self, name): + final_name = self.get_screenshot_destination(name) + self._logger.info("Saving screenshot '{}' to '{}'".format(name, final_name)) + self._driver.save_screenshot(final_name) + + def get_timeout(self): + if "TEST_TIMEOUT" in os.environ.keys(): + return int(os.getenv("TEST_TIMEOUT")) + else: + return 5 + + def maybe_collect_reference(self): + return "TEST_COLLECT_REFERENCE" in os.environ.keys() + + def open_tab(self, url): + opened_tabs = len(self._driver.window_handles) + + self._driver.switch_to.new_window("tab") + self._wait.until(EC.number_of_windows_to_be(opened_tabs + 1)) + self._driver.get(url) + + return self._driver.current_window_handle + + def assert_rendering(self, exp, element_or_driver): + # wait a bit for things to settle down + time.sleep(0.5) + + # Convert as RGB otherwise we cannot get difference + png_bytes = ( + element_or_driver.screenshot_as_png + if isinstance(element_or_driver, WebElement) + else element_or_driver.get_screenshot_as_png() + ) + svg_png = Image.open(io.BytesIO(png_bytes)).convert("RGB") + svg_png_cropped = svg_png.crop((0, 0, svg_png.width - 20, svg_png.height - 20)) + + if self.maybe_collect_reference(): + new_ref = "new_{}".format(exp["reference"]) + new_ref_file = self.get_screenshot_destination(new_ref) + self._logger.info( + "Collecting new reference screenshot: {} => {}".format( + new_ref, new_ref_file + ) + ) + + with open(new_ref_file, "wb") as current_screenshot: + svg_png_cropped.save(current_screenshot) + + return + + svg_ref = Image.open(os.path.join(self._dir, exp["reference"])).convert("RGB") + diff = ImageChops.difference(svg_ref, svg_png_cropped) + + if diff.getbbox() is not None: + buffered = io.BytesIO() + diff.save(buffered, format="PNG") + + if "TEST_DUMP_DIFF" in os.environ.keys(): + diff_b64 = base64.b64encode(buffered.getvalue()) + self._logger.info( + "data:image/png;base64,{}".format(diff_b64.decode("utf-8")) + ) + + with open( + self.get_screenshot_destination("differences.png"), "wb" + ) as diff_screenshot: + diff_screenshot.write(buffered.getvalue()) + + with open( + self.get_screenshot_destination("current_rendering.png"), "wb" + ) as current_screenshot: + svg_png_cropped.save(current_screenshot) + + assert diff.getbbox() is None, "Mismatching screenshots for {}".format( + exp["reference"] + ) + + +class SnapTests(SnapTestsBase): + def __init__(self, exp): + self._dir = "basic_tests" + super(SnapTests, self).__init__(exp) + + def test_about_support(self, exp): + self.open_tab("about:support") + + version_box = self._wait.until( + EC.visibility_of_element_located((By.ID, "version-box")) + ) + self._wait.until(lambda d: len(version_box.text) > 0) + self._logger.info("about:support version: {}".format(version_box.text)) + assert version_box.text == exp["version_box"], "version text should match" + + distributionid_box = self._wait.until( + EC.visibility_of_element_located((By.ID, "distributionid-box")) + ) + self._wait.until(lambda d: len(distributionid_box.text) > 0) + self._logger.info( + "about:support distribution ID: {}".format(distributionid_box.text) + ) + assert ( + distributionid_box.text == exp["distribution_id"] + ), "distribution_id should match" + + windowing_protocol = self._driver.execute_script( + "return document.querySelector('th[data-l10n-id=\"graphics-window-protocol\"').parentNode.lastChild.textContent;" + ) + self._logger.info( + "about:support windowing protocol: {}".format(windowing_protocol) + ) + assert windowing_protocol == "wayland", "windowing protocol should be wayland" + + return True + + def test_about_buildconfig(self, exp): + self.open_tab("about:buildconfig") + + source_link = self._wait.until( + EC.visibility_of_element_located((By.CSS_SELECTOR, "a")) + ) + self._wait.until(lambda d: len(source_link.text) > 0) + self._logger.info("about:buildconfig source: {}".format(source_link.text)) + assert source_link.text.startswith( + exp["source_repo"] + ), "source repo should exists and match" + + build_flags_box = self._wait.until( + EC.visibility_of_element_located((By.CSS_SELECTOR, "p:last-child")) + ) + self._wait.until(lambda d: len(build_flags_box.text) > 0) + self._logger.info("about:support buildflags: {}".format(build_flags_box.text)) + assert ( + build_flags_box.text.find(exp["official"]) >= 0 + ), "official build flag should be there" + + return True + + def test_youtube(self, exp): + self.open_tab("https://www.youtube.com") + + # Wait for the consent dialog and accept it + self._logger.info("Wait for consent form") + try: + self._wait.until( + EC.visibility_of_element_located( + (By.CSS_SELECTOR, "button[aria-label*=Accept]") + ) + ).click() + except TimeoutException: + self._logger.info("Wait for consent form: timed out, maybe it is not here") + + try: + # Find first video and click it + self._logger.info("Wait for one video") + self._wait.until( + EC.visibility_of_element_located((By.ID, "video-title-link")) + ).click() + except TimeoutException: + # We might have got the "try searching to get started" + # link to News channel + self._driver.get("https://www.youtube.com/channel/UCYfdidRxbB8Qhf0Nx7ioOYw") + self._logger.info("Wait again for one video") + self._wait.until( + EC.visibility_of_element_located((By.ID, "video-title-link")) + ).click() + + # Wait for duration to be set to something + self._logger.info("Wait for video to start") + video = self._wait.until( + EC.visibility_of_element_located((By.CLASS_NAME, "html5-main-video")) + ) + self._wait.until(lambda d: type(video.get_property("duration")) == float) + self._logger.info("video duration: {}".format(video.get_property("duration"))) + assert ( + video.get_property("duration") > exp["duration"] + ), "youtube video should have duration" + + self._wait.until(lambda d: video.get_property("currentTime") > exp["playback"]) + self._logger.info("video played: {}".format(video.get_property("currentTime"))) + assert ( + video.get_property("currentTime") > exp["playback"] + ), "youtube video should perform playback" + + return True + + +if __name__ == "__main__": + SnapTests(exp=sys.argv[1]) diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in new file mode 100644 index 0000000000..92e26dd699 --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests/expectations.json.in @@ -0,0 +1,14 @@ +{ + "test_about_support": { + "version_box": "#RUNTIME_VERSION#", + "distribution_id": "canonical-002" + }, + "test_about_buildconfig": { + "source_repo": "https://hg.mozilla.org/", + "official": "MOZILLA_OFFICIAL=1" + }, + "test_youtube": { + "duration": 1, + "playback": 2 + } +} diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py new file mode 100644 index 0000000000..781000981e --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py @@ -0,0 +1,860 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + + +import os +import random +import tempfile +import time + +from basic_tests import SnapTestsBase +from selenium.common.exceptions import StaleElementReferenceException, TimeoutException +from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.select import Select + + +class QATests(SnapTestsBase): + def __init__(self): + self._dir = "qa_tests" + + super(QATests, self).__init__( + exp=os.path.join(self._dir, "qa_expectations.json") + ) + + def _test_audio_playback( + self, url, iframe_selector=None, click_to_play=False, video_selector=None + ): + self._logger.info("open url {}".format(url)) + if url: + self.open_tab(url) + + if iframe_selector: + self._logger.info("find iframe") + iframe = self._driver.find_element(By.CSS_SELECTOR, iframe_selector) + self._driver.switch_to.frame(iframe) + + self._logger.info("find video") + video = self._wait.until( + EC.visibility_of_element_located( + (By.CSS_SELECTOR, video_selector or "video") + ) + ) + self._wait.until(lambda d: type(video.get_property("duration")) == float) + assert video.get_property("duration") > 0.0, "