Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 970 insertions, 0 deletions

diff --git a/gfx/wr/wrench/src/reftest.rs b/gfx/wr/wrench/src/reftest.rs
new file mode 100644
index 0000000000..0e900224b8
--- /dev/null
+++ b/gfx/wr/wrench/src/reftest.rs
@@ -0,0 +1,970 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use crate::{WindowWrapper, NotifierEvent};
+use image::load as load_piston_image;
+use image::png::PNGEncoder;
+use image::{ColorType, ImageFormat};
+use crate::parse_function::parse_function;
+use crate::png::save_flipped;
+use std::{cmp, env};
+use std::fmt::{Display, Error, Formatter};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::mpsc::Receiver;
+use webrender::RenderResults;
+use webrender::api::*;
+use webrender::render_api::*;
+use webrender::api::units::*;
+use crate::wrench::{Wrench, WrenchThing};
+use crate::yaml_frame_reader::YamlFrameReader;
+
+
+const OPTION_DISABLE_SUBPX: &str = "disable-subpixel";
+const OPTION_DISABLE_AA: &str = "disable-aa";
+const OPTION_ALLOW_MIPMAPS: &str = "allow-mipmaps";
+
+pub struct ReftestOptions {
+    // These override values that are lower.
+    pub allow_max_difference: usize,
+    pub allow_num_differences: usize,
+}
+
+impl ReftestOptions {
+    pub fn default() -> Self {
+        ReftestOptions {
+            allow_max_difference: 0,
+            allow_num_differences: 0,
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub enum ReftestOp {
+    /// Expect that the images match the reference
+    Equal,
+    /// Expect that the images *don't* match the reference
+    NotEqual,
+    /// Expect that drawing the reference at different tiles sizes gives the same pixel exact result.
+    Accurate,
+    /// Expect that drawing the reference at different tiles sizes gives a *different* pixel exact result.
+    Inaccurate,
+}
+
+impl Display for ReftestOp {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(
+            f,
+            "{}",
+            match *self {
+                ReftestOp::Equal => "==".to_owned(),
+                ReftestOp::NotEqual => "!=".to_owned(),
+                ReftestOp::Accurate => "**".to_owned(),
+                ReftestOp::Inaccurate => "!*".to_owned(),
+            }
+        )
+    }
+}
+
+#[derive(Debug)]
+enum ExtraCheck {
+    DrawCalls(usize),
+    AlphaTargets(usize),
+    ColorTargets(usize),
+}
+
+impl ExtraCheck {
+    fn run(&self, results: &[RenderResults]) -> bool {
+        match *self {
+            ExtraCheck::DrawCalls(x) =>
+                x == results.last().unwrap().stats.total_draw_calls,
+            ExtraCheck::AlphaTargets(x) =>
+                x == results.last().unwrap().stats.alpha_target_count,
+            ExtraCheck::ColorTargets(x) =>
+                x == results.last().unwrap().stats.color_target_count,
+        }
+    }
+}
+
+pub struct RefTestFuzzy {
+    max_difference: usize,
+    num_differences: usize,
+}
+
+pub struct Reftest {
+    op: ReftestOp,
+    test: Vec<PathBuf>,
+    reference: PathBuf,
+    font_render_mode: Option<FontRenderMode>,
+    fuzziness: Vec<RefTestFuzzy>,
+    extra_checks: Vec<ExtraCheck>,
+    allow_mipmaps: bool,
+    force_subpixel_aa_where_possible: Option<bool>,
+    max_surface_override: Option<usize>,
+}
+
+impl Reftest {
+    /// Check the positive case (expecting equality) and report details if different
+    fn check_and_report_equality_failure(
+        &self,
+        comparison: ReftestImageComparison,
+        test: &ReftestImage,
+        reference: &ReftestImage,
+    ) -> bool {
+        match comparison {
+            ReftestImageComparison::Equal => {
+                true
+            }
+            ReftestImageComparison::NotEqual { difference_histogram, max_difference, count_different } => {
+                // Each entry in the sorted self.fuzziness list represents a bucket which
+                // allows at most num_differences pixels with a difference of at most
+                // max_difference -- but with the caveat that a difference which is small
+                // enough to be less than a max_difference of an earlier bucket, must be
+                // counted against that bucket.
+                //
+                // Thus the test will fail if the number of pixels with a difference
+                // > fuzzy[j-1].max_difference and <= fuzzy[j].max_difference
+                // exceeds fuzzy[j].num_differences.
+                //
+                // (For the first entry, consider fuzzy[j-1] to allow zero pixels of zero
+                // difference).
+                //
+                // For example, say we have this histogram of differences:
+                //
+                //       | [0] [1] [2] [3] [4] [5] [6] ... [255]
+                // ------+------------------------------------------
+                // Hist. |  0   3   2   1   6   2   0  ...   0
+                //
+                // Ie. image comparison found 3 pixels that differ by 1, 2 that differ by 2, etc.
+                // (Note that entry 0 is always zero, we don't count matching pixels.)
+                //
+                // First we calculate an inclusive prefix sum:
+                //
+                //       | [0] [1] [2] [3] [4] [5] [6] ... [255]
+                // ------+------------------------------------------
+                // Hist. |  0   3   2   1   6   2   0  ...   0
+                // Sum   |  0   3   5   6  12  14  14  ...  14
+                //
+                // Let's say the fuzzy statements are:
+                // Fuzzy( 2, 6 )    -- allow up to 6 pixels that differ by 2 or less
+                // Fuzzy( 4, 8 )    -- allow up to 8 pixels that differ by 4 or less _but_
+                //                     also by more than 2 (= by 3 or 4).
+                //
+                // The first  check is Sum[2] <= max 6  which passes: 5 <= 6.
+                // The second check is Sum[4] - Sum[2] <= max 8  which passes: 12-5 <= 8.
+                // Finally we check if there are any pixels that exceed the max difference (4)
+                // by checking Sum[255] - Sum[4] which shows there are 14-12 == 2 so we fail.
+
+                let prefix_sum = difference_histogram.iter()
+                                                     .scan(0, |sum, i| { *sum += i; Some(*sum) })
+                                                     .collect::<Vec<_>>();
+
+                // check each fuzzy statement for violations.
+                assert_eq!(0, difference_histogram[0]);
+                assert_eq!(0, prefix_sum[0]);
+
+                // loop invariant: this is the max_difference of the previous iteration's 'fuzzy'
+                let mut previous_max_diff = 0;
+
+                // loop invariant: this is the number of pixels to ignore as they have been counted
+                // against previous iterations' fuzzy statements.
+                let mut previous_sum_fail = 0;  // ==  prefix_sum[previous_max_diff]
+
+                let mut is_failing = false;
+                let mut fail_text = String::new();
+
+                for fuzzy in &self.fuzziness {
+                    let fuzzy_max_difference = cmp::min(255, fuzzy.max_difference);
+                    let num_differences = prefix_sum[fuzzy_max_difference] - previous_sum_fail;
+                    if num_differences > fuzzy.num_differences {
+                        fail_text.push_str(
+                            &format!("{} differences > {} and <= {} (allowed {}); ",
+                                     num_differences,
+                                     previous_max_diff, fuzzy_max_difference,
+                                     fuzzy.num_differences));
+                        is_failing = true;
+                    }
+                    previous_max_diff = fuzzy_max_difference;
+                    previous_sum_fail = prefix_sum[previous_max_diff];
+                }
+                // do we have any pixels with a difference above the highest allowed
+                // max difference? if so, we fail the test:
+                let num_differences = prefix_sum[255] - previous_sum_fail;
+                if num_differences > 0 {
+                    fail_text.push_str(
+                        &format!("{} num_differences > {} and <= {} (allowed {}); ",
+                                num_differences,
+                                previous_max_diff, 255,
+                                0));
+                    is_failing = true;
+                }
+
+                if is_failing {
+                    println!(
+                        "REFTEST TEST-UNEXPECTED-FAIL | {} | \
+                         image comparison, max difference: {}, number of differing pixels: {} | {}",
+                        self,
+                        max_difference,
+                        count_different,
+                        fail_text,
+                    );
+                    println!("REFTEST   IMAGE 1 (TEST): {}", test.clone().create_data_uri());
+                    println!(
+                        "REFTEST   IMAGE 2 (REFERENCE): {}",
+                        reference.clone().create_data_uri()
+                    );
+                    println!("REFTEST TEST-END | {}", self);
+
+                    false
+                } else {
+                    true
+                }
+            }
+        }
+    }
+
+    /// Report details of the negative case
+    fn report_unexpected_equality(&self) {
+        println!("REFTEST TEST-UNEXPECTED-FAIL | {} | image comparison", self);
+        println!("REFTEST TEST-END | {}", self);
+    }
+}
+
+impl Display for Reftest {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        let paths: Vec<String> = self.test.iter().map(|t| t.display().to_string()).collect();
+        write!(
+            f,
+            "{} {} {}",
+            paths.join(", "),
+            self.op,
+            self.reference.display()
+        )
+    }
+}
+
+#[derive(Clone)]
+pub struct ReftestImage {
+    pub data: Vec<u8>,
+    pub size: DeviceIntSize,
+}
+
+#[derive(Debug, Clone)]
+pub enum ReftestImageComparison {
+    Equal,
+    NotEqual {
+        /// entry[j] = number of pixels with a difference of exactly j
+        difference_histogram: Vec<usize>,
+        max_difference: usize,
+        count_different: usize,
+    },
+}
+
+impl ReftestImage {
+    pub fn compare(&self, other: &ReftestImage) -> ReftestImageComparison {
+        assert_eq!(self.size, other.size);
+        assert_eq!(self.data.len(), other.data.len());
+        assert_eq!(self.data.len() % 4, 0);
+
+        let mut histogram = [0usize; 256];
+        let mut count = 0;
+        let mut max = 0;
+
+        for (a, b) in self.data.chunks(4).zip(other.data.chunks(4)) {
+            if a != b {
+                let pixel_max = a.iter()
+                    .zip(b.iter())
+                    .map(|(x, y)| (*x as isize - *y as isize).abs() as usize)
+                    .max()
+                    .unwrap();
+
+                count += 1;
+                assert!(pixel_max < 256, "pixel values are not 8 bit, update the histogram binning code");
+                // deliberately avoid counting pixels that match --
+                // histogram[0] stays at zero.
+                // this helps our prefix sum later during analysis to
+                // only count actual differences.
+                histogram[pixel_max as usize] += 1;
+                max = cmp::max(max, pixel_max);
+            }
+        }
+
+        if count != 0 {
+            ReftestImageComparison::NotEqual {
+                difference_histogram: histogram.to_vec(),
+                max_difference: max,
+                count_different: count,
+            }
+        } else {
+            ReftestImageComparison::Equal
+        }
+    }
+
+    pub fn create_data_uri(mut self) -> String {
+        let width = self.size.width;
+        let height = self.size.height;
+
+        // flip image vertically (texture is upside down)
+        let orig_pixels = self.data.clone();
+        let stride = width as usize * 4;
+        for y in 0 .. height as usize {
+            let dst_start = y * stride;
+            let src_start = (height as usize - y - 1) * stride;
+            let src_slice = &orig_pixels[src_start .. src_start + stride];
+            (&mut self.data[dst_start .. dst_start + stride])
+                .clone_from_slice(&src_slice[.. stride]);
+        }
+
+        let mut png: Vec<u8> = vec![];
+        {
+            let encoder = PNGEncoder::new(&mut png);
+            encoder
+                .encode(&self.data[..], width as u32, height as u32, ColorType::Rgba8)
+                .expect("Unable to encode PNG!");
+        }
+        let png_base64 = base64::encode(&png);
+        format!("data:image/png;base64,{}", png_base64)
+    }
+}
+
+struct ReftestManifest {
+    reftests: Vec<Reftest>,
+}
+impl ReftestManifest {
+    fn new(manifest: &Path, environment: &ReftestEnvironment, options: &ReftestOptions) -> ReftestManifest {
+        let dir = manifest.parent().unwrap();
+        let f =
+            File::open(manifest).unwrap_or_else(|_| panic!("couldn't open manifest: {}", manifest.display()));
+        let file = BufReader::new(&f);
+
+        let mut reftests = Vec::new();
+
+        for line in file.lines() {
+            let l = line.unwrap();
+
+            // strip the comments
+            let s = &l[0 .. l.find('#').unwrap_or(l.len())];
+            let s = s.trim();
+            if s.is_empty() {
+                continue;
+            }
+
+            let tokens: Vec<&str> = s.split_whitespace().collect();
+
+            let mut fuzziness = Vec::new();
+            let mut op = None;
+            let mut font_render_mode = None;
+            let mut extra_checks = vec![];
+            let mut allow_mipmaps = false;
+            let mut force_subpixel_aa_where_possible = None;
+            let mut max_surface_override = None;
+
+            let mut parse_command = |token: &str| -> bool {
+                match token {
+                    function if function.starts_with("force_subpixel_aa_where_possible(") => {
+                        let (_, args, _) = parse_function(function);
+                        force_subpixel_aa_where_possible = Some(args[0].parse().unwrap());
+                    }
+                    function if function.starts_with("fuzzy-range(") ||
+                                function.starts_with("fuzzy-range-if(") => {
+                        let (_, mut args, _) = parse_function(function);
+                        if function.starts_with("fuzzy-range-if(") {
+                            if !environment.parse_condition(args.remove(0)).expect("unknown condition") {
+                                return true;
+                            }
+                            fuzziness.clear();
+                        }
+                        let num_range = args.len() / 2;
+                        for range in 0..num_range {
+                            let mut max = args[range * 2    ];
+                            let mut num = args[range * 2 + 1];
+                            if max.starts_with("<=") { // trim_start_matches would allow <=<=123
+                                max = &max[2..];
+                            }
+                            if num.starts_with('*') {
+                                num = &num[1..];
+                            }
+                            let max_difference  = max.parse().unwrap();
+                            let num_differences = num.parse().unwrap();
+                            fuzziness.push(RefTestFuzzy { max_difference, num_differences });
+                        }
+                    }
+                    function if function.starts_with("fuzzy(") ||
+                                function.starts_with("fuzzy-if(") => {
+                        let (_, mut args, _) = parse_function(function);
+                        if function.starts_with("fuzzy-if(") {
+                            if !environment.parse_condition(args.remove(0)).expect("unknown condition") {
+                                return true;
+                            }
+                            fuzziness.clear();
+                        }
+                        let max_difference = args[0].parse().unwrap();
+                        let num_differences = args[1].parse().unwrap();
+                        assert!(fuzziness.is_empty()); // if this fires, consider fuzzy-range instead
+                        fuzziness.push(RefTestFuzzy { max_difference, num_differences });
+                    }
+                    function if function.starts_with("draw_calls(") => {
+                        let (_, args, _) = parse_function(function);
+                        extra_checks.push(ExtraCheck::DrawCalls(args[0].parse().unwrap()));
+                    }
+                    function if function.starts_with("alpha_targets(") => {
+                        let (_, args, _) = parse_function(function);
+                        extra_checks.push(ExtraCheck::AlphaTargets(args[0].parse().unwrap()));
+                    }
+                    function if function.starts_with("color_targets(") => {
+                        let (_, args, _) = parse_function(function);
+                        extra_checks.push(ExtraCheck::ColorTargets(args[0].parse().unwrap()));
+                    }
+                    function if function.starts_with("max_surface_size(") => {
+                        let (_, args, _) = parse_function(function);
+                        max_surface_override = Some(args[0].parse().unwrap());
+                    }
+                    options if options.starts_with("options(") => {
+                        let (_, args, _) = parse_function(options);
+                        if args.iter().any(|arg| arg == &OPTION_DISABLE_SUBPX) {
+                            font_render_mode = Some(FontRenderMode::Alpha);
+                        }
+                        if args.iter().any(|arg| arg == &OPTION_DISABLE_AA) {
+                            font_render_mode = Some(FontRenderMode::Mono);
+                        }
+                        if args.iter().any(|arg| arg == &OPTION_ALLOW_MIPMAPS) {
+                            allow_mipmaps = true;
+                        }
+                    }
+                    _ => return false,
+                }
+                true
+            };
+
+            let mut paths = vec![];
+            for (i, token) in tokens.iter().enumerate() {
+                match *token {
+                    "include" => {
+                        assert!(i == 0, "include must be by itself");
+                        let include = dir.join(tokens[1]);
+
+                        reftests.append(
+                            &mut ReftestManifest::new(include.as_path(), environment, options).reftests,
+                        );
+
+                        break;
+                    }
+                    "==" => {
+                        op = Some(ReftestOp::Equal);
+                    }
+                    "!=" => {
+                        op = Some(ReftestOp::NotEqual);
+                    }
+                    "**" => {
+                        op = Some(ReftestOp::Accurate);
+                    }
+                    "!*" => {
+                        op = Some(ReftestOp::Inaccurate);
+                    }
+                    cond if cond.starts_with("if(") => {
+                        let (_, args, _) = parse_function(cond);
+                        if environment.parse_condition(args[0]).expect("unknown condition") {
+                            for command in &args[1..] {
+                                parse_command(command);
+                            }
+                        }
+                    }
+                    command if parse_command(command) => {}
+                    _ => {
+                        match environment.parse_condition(*token) {
+                            Some(true) => {}
+                            Some(false) => break,
+                            _ => paths.push(dir.join(*token)),
+                        }
+                    }
+                }
+            }
+
+            // Don't try to add tests for include lines.
+            if op.is_none() {
+                assert!(paths.is_empty(), "paths = {:?}", paths);
+                continue;
+            }
+            let op = op.unwrap();
+
+            // The reference is the last path provided. If multiple paths are
+            // passed for the test, they render sequentially before being
+            // compared to the reference, which is useful for testing
+            // invalidation.
+            let reference = paths.pop().unwrap();
+            let test = paths;
+
+            if environment.platform == "android" {
+                // Add some fuzz on mobile as we do for non-wrench reftests.
+                // First remove the ranges with difference <= 2, otherwise they might cause the
+                // test to fail before the new range is picked up.
+                fuzziness.retain(|fuzzy| fuzzy.max_difference > 2);
+                fuzziness.push(RefTestFuzzy { max_difference: 2, num_differences: std::usize::MAX });
+            }
+
+            // to avoid changing the meaning of existing tests, the case of
+            // only a single (or no) 'fuzzy' keyword means we use the max
+            // of that fuzzy and options.allow_.. (we don't want that to
+            // turn into a test that allows fuzzy.allow_ *plus* options.allow_):
+            match fuzziness.len() {
+                0 => fuzziness.push(RefTestFuzzy {
+                        max_difference: options.allow_max_difference,
+                        num_differences: options.allow_num_differences }),
+                1 => {
+                    let fuzzy = &mut fuzziness[0];
+                    fuzzy.max_difference = cmp::max(fuzzy.max_difference, options.allow_max_difference);
+                    fuzzy.num_differences = cmp::max(fuzzy.num_differences, options.allow_num_differences);
+                },
+                _ => {
+                    // ignore options, use multiple fuzzy keywords instead. make sure
+                    // the list is sorted to speed up counting violations.
+                    fuzziness.sort_by(|a, b| a.max_difference.cmp(&b.max_difference));
+                    for pair in fuzziness.windows(2) {
+                        if pair[0].max_difference == pair[1].max_difference {
+                            println!("Warning: repeated fuzzy of max_difference {} ignored.",
+                                     pair[1].max_difference);
+                        }
+                    }
+                }
+            }
+
+            reftests.push(Reftest {
+                op,
+                test,
+                reference,
+                font_render_mode,
+                fuzziness,
+                extra_checks,
+                allow_mipmaps,
+                force_subpixel_aa_where_possible,
+                max_surface_override,
+            });
+        }
+
+        ReftestManifest { reftests }
+    }
+
+    fn find(&self, prefix: &Path) -> Vec<&Reftest> {
+        self.reftests
+            .iter()
+            .filter(|x| {
+                x.test.iter().any(|t| t.starts_with(prefix)) || x.reference.starts_with(prefix)
+            })
+            .collect()
+    }
+}
+
+struct YamlRenderOutput {
+    image: ReftestImage,
+    results: RenderResults,
+}
+
+struct ReftestEnvironment {
+    pub platform: &'static str,
+    pub version: Option<semver::Version>,
+    pub mode: &'static str,
+}
+
+impl ReftestEnvironment {
+    fn new(wrench: &Wrench, window: &WindowWrapper) -> Self {
+        Self {
+            platform: Self::platform(wrench, window),
+            version: Self::version(wrench, window),
+            mode: Self::mode(),
+        }
+    }
+
+    fn has(&self, condition: &str) -> bool {
+        if self.platform == condition || self.mode == condition {
+            return true;
+        }
+        if let (Some(v), Ok(r)) = (&self.version, &semver::VersionReq::parse(condition)) {
+            if r.matches(v) {
+                return true;
+            }
+        }
+        let envkey = format!("WRENCH_REFTEST_CONDITION_{}", condition.to_uppercase());
+        env::var(envkey).is_ok()
+    }
+
+    fn platform(_wrench: &Wrench, window: &WindowWrapper) -> &'static str {
+        if window.is_software() {
+            "swgl"
+        } else if cfg!(target_os = "windows") {
+            "win"
+        } else if cfg!(target_os = "linux") {
+            "linux"
+        } else if cfg!(target_os = "macos") {
+            "mac"
+        } else if cfg!(target_os = "android") {
+            "android"
+        } else {
+            "other"
+        }
+    }
+
+    fn version(_wrench: &Wrench, window: &WindowWrapper) -> Option<semver::Version> {
+        if window.is_software() {
+            None
+        } else if cfg!(target_os = "macos") {
+            use std::str;
+            let version_bytes = Command::new("defaults")
+                .arg("read")
+                .arg("loginwindow")
+                .arg("SystemVersionStampAsString")
+                .output()
+                .expect("Failed to get macOS version")
+                .stdout;
+            let mut version_string = str::from_utf8(&version_bytes)
+                .expect("Failed to read macOS version")
+                .trim()
+                .to_string();
+            // On some machines this produces just the major.minor and on
+            // some machines this gives major.minor.patch. But semver requires
+            // the patch so we fake one if it's not there.
+            if version_string.chars().filter(|c| *c == '.').count() == 1 {
+                version_string.push_str(".0");
+            }
+            Some(semver::Version::parse(&version_string)
+                 .unwrap_or_else(|_| panic!("Failed to parse macOS version {}", version_string)))
+        } else {
+            None
+        }
+    }
+
+    fn mode() -> &'static str {
+        if cfg!(debug_assertions) {
+            "debug"
+        } else {
+            "release"
+        }
+    }
+
+    fn parse_condition(&self, token: &str) -> Option<bool> {
+        match token {
+            platform if platform.starts_with("skip_on(") => {
+                // e.g. skip_on(android,debug) will skip only when
+                // running on a debug android build.
+                let (_, args, _) = parse_function(platform);
+                Some(!args.iter().all(|arg| self.has(arg)))
+            }
+            platform if platform.starts_with("env(") => {
+                // non-negated version of skip_on for nested conditions
+                let (_, args, _) = parse_function(platform);
+                Some(args.iter().all(|arg| self.has(arg)))
+            }
+            platform if platform.starts_with("platform(") => {
+                let (_, args, _) = parse_function(platform);
+                // Skip due to platform not matching
+                Some(args.iter().any(|arg| arg == &self.platform))
+            }
+            op if op.starts_with("not(") => {
+                let (_, args, _) = parse_function(op);
+                Some(!self.parse_condition(args[0]).expect("unknown condition"))
+            }
+            op if op.starts_with("or(") => {
+                let (_, args, _) = parse_function(op);
+                Some(args.iter().any(|arg| self.parse_condition(arg).expect("unknown condition")))
+            }
+            op if op.starts_with("and(") => {
+                let (_, args, _) = parse_function(op);
+                Some(args.iter().all(|arg| self.parse_condition(arg).expect("unknown condition")))
+            }
+            _ => None,
+        }
+    }
+}
+
+pub struct ReftestHarness<'a> {
+    wrench: &'a mut Wrench,
+    window: &'a mut WindowWrapper,
+    rx: &'a Receiver<NotifierEvent>,
+    environment: ReftestEnvironment,
+}
+impl<'a> ReftestHarness<'a> {
+    pub fn new(wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: &'a Receiver<NotifierEvent>) -> Self {
+        let environment = ReftestEnvironment::new(wrench, window);
+        ReftestHarness { wrench, window, rx, environment }
+    }
+
+    pub fn run(mut self, base_manifest: &Path, reftests: Option<&Path>, options: &ReftestOptions) -> usize {
+        let manifest = ReftestManifest::new(base_manifest, &self.environment, options);
+        let reftests = manifest.find(reftests.unwrap_or(&PathBuf::new()));
+
+        let mut total_passing = 0;
+        let mut failing = Vec::new();
+
+        for t in reftests {
+            if self.run_reftest(t) {
+                total_passing += 1;
+            } else {
+                failing.push(t);
+            }
+        }
+
+        println!(
+            "REFTEST INFO | {} passing, {} failing",
+            total_passing,
+            failing.len()
+        );
+
+        if !failing.is_empty() {
+            println!("\nReftests with unexpected results:");
+
+            for reftest in &failing {
+                println!("\t{}", reftest);
+            }
+        }
+
+        failing.len()
+    }
+
+    fn run_reftest(&mut self, t: &Reftest) -> bool {
+        let test_name = t.to_string();
+        println!("REFTEST {}", test_name);
+        profile_scope!("wrench reftest", text: &test_name);
+
+        self.wrench
+            .api
+            .send_debug_cmd(
+                DebugCommand::ClearCaches(ClearCache::all())
+            );
+
+        let quality_settings = QualitySettings {
+            force_subpixel_aa_where_possible: t.force_subpixel_aa_where_possible.unwrap_or_default(),
+        };
+
+        self.wrench.set_quality_settings(quality_settings);
+
+        if let Some(max_surface_override) = t.max_surface_override {
+            self.wrench
+                .api
+                .send_debug_cmd(
+                    DebugCommand::SetMaximumSurfaceSize(Some(max_surface_override))
+                );
+        }
+
+        let window_size = self.window.get_inner_size();
+        let reference_image = match t.reference.extension().unwrap().to_str().unwrap() {
+            "yaml" => None,
+            "png" => Some(self.load_image(t.reference.as_path(), ImageFormat::Png)),
+            other => panic!("Unknown reftest extension: {}", other),
+        };
+        let test_size = reference_image.as_ref().map_or(window_size, |img| img.size);
+
+        // The reference can be smaller than the window size, in which case
+        // we only compare the intersection.
+        //
+        // Note also that, when we have multiple test scenes in sequence, we
+        // want to test the picture caching machinery. But since picture caching
+        // only takes effect after the result has been the same several frames in
+        // a row, we need to render the scene multiple times.
+        let mut images = vec![];
+        let mut results = vec![];
+
+        match t.op {
+            ReftestOp::Equal | ReftestOp::NotEqual => {
+                // For equality tests, render each test image and store result
+                for filename in t.test.iter() {
+                    let output = self.render_yaml(
+                        filename,
+                        test_size,
+                        t.font_render_mode,
+                        t.allow_mipmaps,
+                    );
+                    images.push(output.image);
+                    results.push(output.results);
+                }
+            }
+            ReftestOp::Accurate | ReftestOp::Inaccurate => {
+                // For accuracy tests, render the reference yaml at an arbitrary series
+                // of tile sizes, and compare to the reference drawn at normal tile size.
+                let tile_sizes = [
+                    DeviceIntSize::new(128, 128),
+                    DeviceIntSize::new(256, 256),
+                    DeviceIntSize::new(512, 512),
+                ];
+
+                for tile_size in &tile_sizes {
+                    self.wrench
+                        .api
+                        .send_debug_cmd(
+                            DebugCommand::SetPictureTileSize(Some(*tile_size))
+                        );
+
+                    let output = self.render_yaml(
+                        &t.reference,
+                        test_size,
+                        t.font_render_mode,
+                        t.allow_mipmaps,
+                    );
+                    images.push(output.image);
+                    results.push(output.results);
+                }
+
+                self.wrench
+                    .api
+                    .send_debug_cmd(
+                        DebugCommand::SetPictureTileSize(None)
+                    );
+            }
+        }
+
+        let reference = if let Some(image) = reference_image {
+            let save_all_png = false; // flip to true to update all the tests!
+            if save_all_png {
+                let img = images.last().unwrap();
+                save_flipped(&t.reference, img.data.clone(), img.size);
+            }
+            image
+        } else {
+            let output = self.render_yaml(
+                &t.reference,
+                test_size,
+                t.font_render_mode,
+                t.allow_mipmaps,
+            );
+            output.image
+        };
+
+        if let Some(_) = t.max_surface_override {
+            self.wrench
+                .api
+                .send_debug_cmd(
+                    DebugCommand::SetMaximumSurfaceSize(None)
+                );
+        }
+
+        for extra_check in t.extra_checks.iter() {
+            if !extra_check.run(&results) {
+                println!(
+                    "REFTEST TEST-UNEXPECTED-FAIL | {} | Failing Check: {:?} | Actual Results: {:?}",
+                    t,
+                    extra_check,
+                    results,
+                );
+                println!("REFTEST TEST-END | {}", t);
+                return false;
+            }
+        }
+
+        match t.op {
+            ReftestOp::Equal => {
+                // Ensure that the final image matches the reference
+                let test = images.pop().unwrap();
+                let comparison = test.compare(&reference);
+                t.check_and_report_equality_failure(
+                    comparison,
+                    &test,
+                    &reference,
+                )
+            }
+            ReftestOp::NotEqual => {
+                // Ensure that the final image *doesn't* match the reference
+                let test = images.pop().unwrap();
+                let comparison = test.compare(&reference);
+                match comparison {
+                    ReftestImageComparison::Equal => {
+                        t.report_unexpected_equality();
+                        false
+                    }
+                    ReftestImageComparison::NotEqual { .. } => {
+                        true
+                    }
+                }
+            }
+            ReftestOp::Accurate => {
+                // Ensure that *all* images match the reference
+                for test in images.drain(..) {
+                    let comparison = test.compare(&reference);
+
+                    if !t.check_and_report_equality_failure(
+                        comparison,
+                        &test,
+                        &reference,
+                    ) {
+                        return false;
+                    }
+                }
+
+                true
+            }
+            ReftestOp::Inaccurate => {
+                // Ensure that at least one of the images doesn't match the reference
+                let all_same = images.iter().all(|image| {
+                    match image.compare(&reference) {
+                        ReftestImageComparison::Equal => true,
+                        ReftestImageComparison::NotEqual { .. } => false,
+                    }
+                });
+
+                if all_same {
+                    t.report_unexpected_equality();
+                }
+
+                !all_same
+            }
+        }
+    }
+
+    fn load_image(&mut self, filename: &Path, format: ImageFormat) -> ReftestImage {
+        let file = BufReader::new(File::open(filename).unwrap());
+        let img_raw = load_piston_image(file, format).unwrap();
+        let img = img_raw.flipv().to_rgba();
+        let size = img.dimensions();
+        ReftestImage {
+            data: img.into_raw(),
+            size: DeviceIntSize::new(size.0 as i32, size.1 as i32),
+        }
+    }
+
+    fn render_yaml(
+        &mut self,
+        filename: &Path,
+        size: DeviceIntSize,
+        font_render_mode: Option<FontRenderMode>,
+        allow_mipmaps: bool,
+    ) -> YamlRenderOutput {
+        let mut reader = YamlFrameReader::new(filename);
+        reader.set_font_render_mode(font_render_mode);
+        reader.allow_mipmaps(allow_mipmaps);
+        reader.do_frame(self.wrench);
+
+        self.wrench.api.flush_scene_builder();
+
+        // wait for the frame
+        self.rx.recv().unwrap();
+        let results = self.wrench.render();
+
+        let window_size = self.window.get_inner_size();
+        assert!(
+            size.width <= window_size.width &&
+            size.height <= window_size.height,
+            "size={:?} ws={:?}", size, window_size
+        );
+
+        // taking the bottom left sub-rectangle
+        let rect = FramebufferIntRect::from_origin_and_size(
+            FramebufferIntPoint::new(0, window_size.height - size.height),
+            FramebufferIntSize::new(size.width, size.height),
+        );
+        let pixels = self.wrench.renderer.read_pixels_rgba8(rect);
+        self.window.swap_buffers();
+
+        let write_debug_images = false;
+        if write_debug_images {
+            let debug_path = filename.with_extension("yaml.png");
+            save_flipped(debug_path, pixels.clone(), size);
+        }
+
+        reader.deinit(self.wrench);
+
+        YamlRenderOutput {
+            image: ReftestImage { data: pixels, size },
+            results,
+        }
+    }
+}