summaryrefslogtreecommitdiffstats
path: root/gfx/wr/wrench/src/reftest.rs
diff options
context:
space:
mode:
Diffstat (limited to 'gfx/wr/wrench/src/reftest.rs')
-rw-r--r--gfx/wr/wrench/src/reftest.rs970
1 files changed, 970 insertions, 0 deletions
diff --git a/gfx/wr/wrench/src/reftest.rs b/gfx/wr/wrench/src/reftest.rs
new file mode 100644
index 0000000000..136a447720
--- /dev/null
+++ b/gfx/wr/wrench/src/reftest.rs
@@ -0,0 +1,970 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use crate::{WindowWrapper, NotifierEvent};
+use image::load as load_piston_image;
+use image::png::PNGEncoder;
+use image::{ColorType, ImageFormat};
+use crate::parse_function::parse_function;
+use crate::png::save_flipped;
+use std::{cmp, env};
+use std::fmt::{Display, Error, Formatter};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::mpsc::Receiver;
+use webrender::RenderResults;
+use webrender::api::*;
+use webrender::render_api::*;
+use webrender::api::units::*;
+use crate::wrench::{Wrench, WrenchThing};
+use crate::yaml_frame_reader::YamlFrameReader;
+
+
+const OPTION_DISABLE_SUBPX: &str = "disable-subpixel";
+const OPTION_DISABLE_AA: &str = "disable-aa";
+const OPTION_ALLOW_MIPMAPS: &str = "allow-mipmaps";
+
+pub struct ReftestOptions {
+ // These override values that are lower.
+ pub allow_max_difference: usize,
+ pub allow_num_differences: usize,
+}
+
+impl ReftestOptions {
+ pub fn default() -> Self {
+ ReftestOptions {
+ allow_max_difference: 0,
+ allow_num_differences: 0,
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub enum ReftestOp {
+ /// Expect that the images match the reference
+ Equal,
+ /// Expect that the images *don't* match the reference
+ NotEqual,
+ /// Expect that drawing the reference at different tiles sizes gives the same pixel exact result.
+ Accurate,
+ /// Expect that drawing the reference at different tiles sizes gives a *different* pixel exact result.
+ Inaccurate,
+}
+
+impl Display for ReftestOp {
+ fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+ write!(
+ f,
+ "{}",
+ match *self {
+ ReftestOp::Equal => "==".to_owned(),
+ ReftestOp::NotEqual => "!=".to_owned(),
+ ReftestOp::Accurate => "**".to_owned(),
+ ReftestOp::Inaccurate => "!*".to_owned(),
+ }
+ )
+ }
+}
+
+#[derive(Debug)]
+enum ExtraCheck {
+ DrawCalls(usize),
+ AlphaTargets(usize),
+ ColorTargets(usize),
+}
+
+impl ExtraCheck {
+ fn run(&self, results: &[RenderResults]) -> bool {
+ match *self {
+ ExtraCheck::DrawCalls(x) =>
+ x == results.last().unwrap().stats.total_draw_calls,
+ ExtraCheck::AlphaTargets(x) =>
+ x == results.last().unwrap().stats.alpha_target_count,
+ ExtraCheck::ColorTargets(x) =>
+ x == results.last().unwrap().stats.color_target_count,
+ }
+ }
+}
+
+pub struct RefTestFuzzy {
+ max_difference: usize,
+ num_differences: usize,
+}
+
+pub struct Reftest {
+ op: ReftestOp,
+ test: Vec<PathBuf>,
+ reference: PathBuf,
+ font_render_mode: Option<FontRenderMode>,
+ fuzziness: Vec<RefTestFuzzy>,
+ extra_checks: Vec<ExtraCheck>,
+ allow_mipmaps: bool,
+ force_subpixel_aa_where_possible: Option<bool>,
+ max_surface_override: Option<usize>,
+}
+
+impl Reftest {
+ /// Check the positive case (expecting equality) and report details if different
+ fn check_and_report_equality_failure(
+ &self,
+ comparison: ReftestImageComparison,
+ test: &ReftestImage,
+ reference: &ReftestImage,
+ ) -> bool {
+ match comparison {
+ ReftestImageComparison::Equal => {
+ true
+ }
+ ReftestImageComparison::NotEqual { difference_histogram, max_difference, count_different } => {
+ // Each entry in the sorted self.fuzziness list represents a bucket which
+ // allows at most num_differences pixels with a difference of at most
+ // max_difference -- but with the caveat that a difference which is small
+ // enough to be less than a max_difference of an earlier bucket, must be
+ // counted against that bucket.
+ //
+ // Thus the test will fail if the number of pixels with a difference
+ // > fuzzy[j-1].max_difference and <= fuzzy[j].max_difference
+ // exceeds fuzzy[j].num_differences.
+ //
+ // (For the first entry, consider fuzzy[j-1] to allow zero pixels of zero
+ // difference).
+ //
+ // For example, say we have this histogram of differences:
+ //
+ // | [0] [1] [2] [3] [4] [5] [6] ... [255]
+ // ------+------------------------------------------
+ // Hist. | 0 3 2 1 6 2 0 ... 0
+ //
+ // Ie. image comparison found 3 pixels that differ by 1, 2 that differ by 2, etc.
+ // (Note that entry 0 is always zero, we don't count matching pixels.)
+ //
+ // First we calculate an inclusive prefix sum:
+ //
+ // | [0] [1] [2] [3] [4] [5] [6] ... [255]
+ // ------+------------------------------------------
+ // Hist. | 0 3 2 1 6 2 0 ... 0
+ // Sum | 0 3 5 6 12 14 14 ... 14
+ //
+ // Let's say the fuzzy statements are:
+ // Fuzzy( 2, 6 ) -- allow up to 6 pixels that differ by 2 or less
+ // Fuzzy( 4, 8 ) -- allow up to 8 pixels that differ by 4 or less _but_
+ // also by more than 2 (= by 3 or 4).
+ //
+ // The first check is Sum[2] <= max 6 which passes: 5 <= 6.
+ // The second check is Sum[4] - Sum[2] <= max 8 which passes: 12-5 <= 8.
+ // Finally we check if there are any pixels that exceed the max difference (4)
+ // by checking Sum[255] - Sum[4] which shows there are 14-12 == 2 so we fail.
+
+ let prefix_sum = difference_histogram.iter()
+ .scan(0, |sum, i| { *sum += i; Some(*sum) })
+ .collect::<Vec<_>>();
+
+ // check each fuzzy statement for violations.
+ assert_eq!(0, difference_histogram[0]);
+ assert_eq!(0, prefix_sum[0]);
+
+ // loop invariant: this is the max_difference of the previous iteration's 'fuzzy'
+ let mut previous_max_diff = 0;
+
+ // loop invariant: this is the number of pixels to ignore as they have been counted
+ // against previous iterations' fuzzy statements.
+ let mut previous_sum_fail = 0; // == prefix_sum[previous_max_diff]
+
+ let mut is_failing = false;
+ let mut fail_text = String::new();
+
+ for fuzzy in &self.fuzziness {
+ let fuzzy_max_difference = cmp::min(255, fuzzy.max_difference);
+ let num_differences = prefix_sum[fuzzy_max_difference] - previous_sum_fail;
+ if num_differences > fuzzy.num_differences {
+ fail_text.push_str(
+ &format!("{} differences > {} and <= {} (allowed {}); ",
+ num_differences,
+ previous_max_diff, fuzzy_max_difference,
+ fuzzy.num_differences));
+ is_failing = true;
+ }
+ previous_max_diff = fuzzy_max_difference;
+ previous_sum_fail = prefix_sum[previous_max_diff];
+ }
+ // do we have any pixels with a difference above the highest allowed
+ // max difference? if so, we fail the test:
+ let num_differences = prefix_sum[255] - previous_sum_fail;
+ if num_differences > 0 {
+ fail_text.push_str(
+ &format!("{} num_differences > {} and <= {} (allowed {}); ",
+ num_differences,
+ previous_max_diff, 255,
+ 0));
+ is_failing = true;
+ }
+
+ if is_failing {
+ println!(
+ "REFTEST TEST-UNEXPECTED-FAIL | {} | \
+ image comparison, max difference: {}, number of differing pixels: {} | {}",
+ self,
+ max_difference,
+ count_different,
+ fail_text,
+ );
+ println!("REFTEST IMAGE 1 (TEST): {}", test.clone().create_data_uri());
+ println!(
+ "REFTEST IMAGE 2 (REFERENCE): {}",
+ reference.clone().create_data_uri()
+ );
+ println!("REFTEST TEST-END | {}", self);
+
+ false
+ } else {
+ true
+ }
+ }
+ }
+ }
+
+ /// Report details of the negative case
+ fn report_unexpected_equality(&self) {
+ println!("REFTEST TEST-UNEXPECTED-FAIL | {} | image comparison", self);
+ println!("REFTEST TEST-END | {}", self);
+ }
+}
+
+impl Display for Reftest {
+ fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+ let paths: Vec<String> = self.test.iter().map(|t| t.display().to_string()).collect();
+ write!(
+ f,
+ "{} {} {}",
+ paths.join(", "),
+ self.op,
+ self.reference.display()
+ )
+ }
+}
+
+#[derive(Clone)]
+pub struct ReftestImage {
+ pub data: Vec<u8>,
+ pub size: DeviceIntSize,
+}
+
+#[derive(Debug, Clone)]
+pub enum ReftestImageComparison {
+ Equal,
+ NotEqual {
+ /// entry[j] = number of pixels with a difference of exactly j
+ difference_histogram: Vec<usize>,
+ max_difference: usize,
+ count_different: usize,
+ },
+}
+
+impl ReftestImage {
+ pub fn compare(&self, other: &ReftestImage) -> ReftestImageComparison {
+ assert_eq!(self.size, other.size);
+ assert_eq!(self.data.len(), other.data.len());
+ assert_eq!(self.data.len() % 4, 0);
+
+ let mut histogram = [0usize; 256];
+ let mut count = 0;
+ let mut max = 0;
+
+ for (a, b) in self.data.chunks(4).zip(other.data.chunks(4)) {
+ if a != b {
+ let pixel_max = a.iter()
+ .zip(b.iter())
+ .map(|(x, y)| (*x as isize - *y as isize).abs() as usize)
+ .max()
+ .unwrap();
+
+ count += 1;
+ assert!(pixel_max < 256, "pixel values are not 8 bit, update the histogram binning code");
+ // deliberately avoid counting pixels that match --
+ // histogram[0] stays at zero.
+ // this helps our prefix sum later during analysis to
+ // only count actual differences.
+ histogram[pixel_max as usize] += 1;
+ max = cmp::max(max, pixel_max);
+ }
+ }
+
+ if count != 0 {
+ ReftestImageComparison::NotEqual {
+ difference_histogram: histogram.to_vec(),
+ max_difference: max,
+ count_different: count,
+ }
+ } else {
+ ReftestImageComparison::Equal
+ }
+ }
+
+ pub fn create_data_uri(mut self) -> String {
+ let width = self.size.width;
+ let height = self.size.height;
+
+ // flip image vertically (texture is upside down)
+ let orig_pixels = self.data.clone();
+ let stride = width as usize * 4;
+ for y in 0 .. height as usize {
+ let dst_start = y * stride;
+ let src_start = (height as usize - y - 1) * stride;
+ let src_slice = &orig_pixels[src_start .. src_start + stride];
+ (&mut self.data[dst_start .. dst_start + stride])
+ .clone_from_slice(&src_slice[.. stride]);
+ }
+
+ let mut png: Vec<u8> = vec![];
+ {
+ let encoder = PNGEncoder::new(&mut png);
+ encoder
+ .encode(&self.data[..], width as u32, height as u32, ColorType::Rgba8)
+ .expect("Unable to encode PNG!");
+ }
+ let png_base64 = base64::encode(&png);
+ format!("data:image/png;base64,{}", png_base64)
+ }
+}
+
+struct ReftestManifest {
+ reftests: Vec<Reftest>,
+}
+impl ReftestManifest {
+ fn new(manifest: &Path, environment: &ReftestEnvironment, options: &ReftestOptions) -> ReftestManifest {
+ let dir = manifest.parent().unwrap();
+ let f =
+ File::open(manifest).unwrap_or_else(|_| panic!("couldn't open manifest: {}", manifest.display()));
+ let file = BufReader::new(&f);
+
+ let mut reftests = Vec::new();
+
+ for line in file.lines() {
+ let l = line.unwrap();
+
+ // strip the comments
+ let s = &l[0 .. l.find('#').unwrap_or(l.len())];
+ let s = s.trim();
+ if s.is_empty() {
+ continue;
+ }
+
+ let tokens: Vec<&str> = s.split_whitespace().collect();
+
+ let mut fuzziness = Vec::new();
+ let mut op = None;
+ let mut font_render_mode = None;
+ let mut extra_checks = vec![];
+ let mut allow_mipmaps = false;
+ let mut force_subpixel_aa_where_possible = None;
+ let mut max_surface_override = None;
+
+ let mut parse_command = |token: &str| -> bool {
+ match token {
+ function if function.starts_with("force_subpixel_aa_where_possible(") => {
+ let (_, args, _) = parse_function(function);
+ force_subpixel_aa_where_possible = Some(args[0].parse().unwrap());
+ }
+ function if function.starts_with("fuzzy-range(") ||
+ function.starts_with("fuzzy-range-if(") => {
+ let (_, mut args, _) = parse_function(function);
+ if function.starts_with("fuzzy-range-if(") {
+ if !environment.parse_condition(args.remove(0)).expect("unknown condition") {
+ return true;
+ }
+ fuzziness.clear();
+ }
+ let num_range = args.len() / 2;
+ for range in 0..num_range {
+ let mut max = args[range * 2 ];
+ let mut num = args[range * 2 + 1];
+ if max.starts_with("<=") { // trim_start_matches would allow <=<=123
+ max = &max[2..];
+ }
+ if num.starts_with('*') {
+ num = &num[1..];
+ }
+ let max_difference = max.parse().unwrap();
+ let num_differences = num.parse().unwrap();
+ fuzziness.push(RefTestFuzzy { max_difference, num_differences });
+ }
+ }
+ function if function.starts_with("fuzzy(") ||
+ function.starts_with("fuzzy-if(") => {
+ let (_, mut args, _) = parse_function(function);
+ if function.starts_with("fuzzy-if(") {
+ if !environment.parse_condition(args.remove(0)).expect("unknown condition") {
+ return true;
+ }
+ fuzziness.clear();
+ }
+ let max_difference = args[0].parse().unwrap();
+ let num_differences = args[1].parse().unwrap();
+ assert!(fuzziness.is_empty()); // if this fires, consider fuzzy-range instead
+ fuzziness.push(RefTestFuzzy { max_difference, num_differences });
+ }
+ function if function.starts_with("draw_calls(") => {
+ let (_, args, _) = parse_function(function);
+ extra_checks.push(ExtraCheck::DrawCalls(args[0].parse().unwrap()));
+ }
+ function if function.starts_with("alpha_targets(") => {
+ let (_, args, _) = parse_function(function);
+ extra_checks.push(ExtraCheck::AlphaTargets(args[0].parse().unwrap()));
+ }
+ function if function.starts_with("color_targets(") => {
+ let (_, args, _) = parse_function(function);
+ extra_checks.push(ExtraCheck::ColorTargets(args[0].parse().unwrap()));
+ }
+ function if function.starts_with("max_surface_size(") => {
+ let (_, args, _) = parse_function(function);
+ max_surface_override = Some(args[0].parse().unwrap());
+ }
+ options if options.starts_with("options(") => {
+ let (_, args, _) = parse_function(options);
+ if args.iter().any(|arg| arg == &OPTION_DISABLE_SUBPX) {
+ font_render_mode = Some(FontRenderMode::Alpha);
+ }
+ if args.iter().any(|arg| arg == &OPTION_DISABLE_AA) {
+ font_render_mode = Some(FontRenderMode::Mono);
+ }
+ if args.iter().any(|arg| arg == &OPTION_ALLOW_MIPMAPS) {
+ allow_mipmaps = true;
+ }
+ }
+ _ => return false,
+ }
+ true
+ };
+
+ let mut paths = vec![];
+ for (i, token) in tokens.iter().enumerate() {
+ match *token {
+ "include" => {
+ assert!(i == 0, "include must be by itself");
+ let include = dir.join(tokens[1]);
+
+ reftests.append(
+ &mut ReftestManifest::new(include.as_path(), environment, options).reftests,
+ );
+
+ break;
+ }
+ "==" => {
+ op = Some(ReftestOp::Equal);
+ }
+ "!=" => {
+ op = Some(ReftestOp::NotEqual);
+ }
+ "**" => {
+ op = Some(ReftestOp::Accurate);
+ }
+ "!*" => {
+ op = Some(ReftestOp::Inaccurate);
+ }
+ cond if cond.starts_with("if(") => {
+ let (_, args, _) = parse_function(cond);
+ if environment.parse_condition(args[0]).expect("unknown condition") {
+ for command in &args[1..] {
+ parse_command(command);
+ }
+ }
+ }
+ command if parse_command(command) => {}
+ _ => {
+ match environment.parse_condition(*token) {
+ Some(true) => {}
+ Some(false) => break,
+ _ => paths.push(dir.join(*token)),
+ }
+ }
+ }
+ }
+
+ // Don't try to add tests for include lines.
+ if op.is_none() {
+ assert!(paths.is_empty(), "paths = {:?}", paths);
+ continue;
+ }
+ let op = op.unwrap();
+
+ // The reference is the last path provided. If multiple paths are
+ // passed for the test, they render sequentially before being
+ // compared to the reference, which is useful for testing
+ // invalidation.
+ let reference = paths.pop().unwrap();
+ let test = paths;
+
+ if environment.platform == "android" {
+ // Add some fuzz on mobile as we do for non-wrench reftests.
+ // First remove the ranges with difference <= 2, otherwise they might cause the
+ // test to fail before the new range is picked up.
+ fuzziness.retain(|fuzzy| fuzzy.max_difference > 2);
+ fuzziness.push(RefTestFuzzy { max_difference: 2, num_differences: std::usize::MAX });
+ }
+
+ // to avoid changing the meaning of existing tests, the case of
+ // only a single (or no) 'fuzzy' keyword means we use the max
+ // of that fuzzy and options.allow_.. (we don't want that to
+ // turn into a test that allows fuzzy.allow_ *plus* options.allow_):
+ match fuzziness.len() {
+ 0 => fuzziness.push(RefTestFuzzy {
+ max_difference: options.allow_max_difference,
+ num_differences: options.allow_num_differences }),
+ 1 => {
+ let mut fuzzy = &mut fuzziness[0];
+ fuzzy.max_difference = cmp::max(fuzzy.max_difference, options.allow_max_difference);
+ fuzzy.num_differences = cmp::max(fuzzy.num_differences, options.allow_num_differences);
+ },
+ _ => {
+ // ignore options, use multiple fuzzy keywords instead. make sure
+ // the list is sorted to speed up counting violations.
+ fuzziness.sort_by(|a, b| a.max_difference.cmp(&b.max_difference));
+ for pair in fuzziness.windows(2) {
+ if pair[0].max_difference == pair[1].max_difference {
+ println!("Warning: repeated fuzzy of max_difference {} ignored.",
+ pair[1].max_difference);
+ }
+ }
+ }
+ }
+
+ reftests.push(Reftest {
+ op,
+ test,
+ reference,
+ font_render_mode,
+ fuzziness,
+ extra_checks,
+ allow_mipmaps,
+ force_subpixel_aa_where_possible,
+ max_surface_override,
+ });
+ }
+
+ ReftestManifest { reftests }
+ }
+
+ fn find(&self, prefix: &Path) -> Vec<&Reftest> {
+ self.reftests
+ .iter()
+ .filter(|x| {
+ x.test.iter().any(|t| t.starts_with(prefix)) || x.reference.starts_with(prefix)
+ })
+ .collect()
+ }
+}
+
+struct YamlRenderOutput {
+ image: ReftestImage,
+ results: RenderResults,
+}
+
+struct ReftestEnvironment {
+ pub platform: &'static str,
+ pub version: Option<semver::Version>,
+ pub mode: &'static str,
+}
+
+impl ReftestEnvironment {
+ fn new(wrench: &Wrench, window: &WindowWrapper) -> Self {
+ Self {
+ platform: Self::platform(wrench, window),
+ version: Self::version(wrench, window),
+ mode: Self::mode(),
+ }
+ }
+
+ fn has(&self, condition: &str) -> bool {
+ if self.platform == condition || self.mode == condition {
+ return true;
+ }
+ if let (Some(v), Ok(r)) = (&self.version, &semver::VersionReq::parse(condition)) {
+ if r.matches(v) {
+ return true;
+ }
+ }
+ let envkey = format!("WRENCH_REFTEST_CONDITION_{}", condition.to_uppercase());
+ env::var(envkey).is_ok()
+ }
+
+ fn platform(_wrench: &Wrench, window: &WindowWrapper) -> &'static str {
+ if window.is_software() {
+ "swgl"
+ } else if cfg!(target_os = "windows") {
+ "win"
+ } else if cfg!(target_os = "linux") {
+ "linux"
+ } else if cfg!(target_os = "macos") {
+ "mac"
+ } else if cfg!(target_os = "android") {
+ "android"
+ } else {
+ "other"
+ }
+ }
+
+ fn version(_wrench: &Wrench, window: &WindowWrapper) -> Option<semver::Version> {
+ if window.is_software() {
+ None
+ } else if cfg!(target_os = "macos") {
+ use std::str;
+ let version_bytes = Command::new("defaults")
+ .arg("read")
+ .arg("loginwindow")
+ .arg("SystemVersionStampAsString")
+ .output()
+ .expect("Failed to get macOS version")
+ .stdout;
+ let mut version_string = str::from_utf8(&version_bytes)
+ .expect("Failed to read macOS version")
+ .trim()
+ .to_string();
+ // On some machines this produces just the major.minor and on
+ // some machines this gives major.minor.patch. But semver requires
+ // the patch so we fake one if it's not there.
+ if version_string.chars().filter(|c| *c == '.').count() == 1 {
+ version_string.push_str(".0");
+ }
+ Some(semver::Version::parse(&version_string)
+ .unwrap_or_else(|_| panic!("Failed to parse macOS version {}", version_string)))
+ } else {
+ None
+ }
+ }
+
+ fn mode() -> &'static str {
+ if cfg!(debug_assertions) {
+ "debug"
+ } else {
+ "release"
+ }
+ }
+
+ fn parse_condition(&self, token: &str) -> Option<bool> {
+ match token {
+ platform if platform.starts_with("skip_on(") => {
+ // e.g. skip_on(android,debug) will skip only when
+ // running on a debug android build.
+ let (_, args, _) = parse_function(platform);
+ Some(!args.iter().all(|arg| self.has(arg)))
+ }
+ platform if platform.starts_with("env(") => {
+ // non-negated version of skip_on for nested conditions
+ let (_, args, _) = parse_function(platform);
+ Some(args.iter().all(|arg| self.has(arg)))
+ }
+ platform if platform.starts_with("platform(") => {
+ let (_, args, _) = parse_function(platform);
+ // Skip due to platform not matching
+ Some(args.iter().any(|arg| arg == &self.platform))
+ }
+ op if op.starts_with("not(") => {
+ let (_, args, _) = parse_function(op);
+ Some(!self.parse_condition(args[0]).expect("unknown condition"))
+ }
+ op if op.starts_with("or(") => {
+ let (_, args, _) = parse_function(op);
+ Some(args.iter().any(|arg| self.parse_condition(arg).expect("unknown condition")))
+ }
+ op if op.starts_with("and(") => {
+ let (_, args, _) = parse_function(op);
+ Some(args.iter().all(|arg| self.parse_condition(arg).expect("unknown condition")))
+ }
+ _ => None,
+ }
+ }
+}
+
+pub struct ReftestHarness<'a> {
+ wrench: &'a mut Wrench,
+ window: &'a mut WindowWrapper,
+ rx: &'a Receiver<NotifierEvent>,
+ environment: ReftestEnvironment,
+}
+impl<'a> ReftestHarness<'a> {
+ pub fn new(wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: &'a Receiver<NotifierEvent>) -> Self {
+ let environment = ReftestEnvironment::new(wrench, window);
+ ReftestHarness { wrench, window, rx, environment }
+ }
+
+ pub fn run(mut self, base_manifest: &Path, reftests: Option<&Path>, options: &ReftestOptions) -> usize {
+ let manifest = ReftestManifest::new(base_manifest, &self.environment, options);
+ let reftests = manifest.find(reftests.unwrap_or(&PathBuf::new()));
+
+ let mut total_passing = 0;
+ let mut failing = Vec::new();
+
+ for t in reftests {
+ if self.run_reftest(t) {
+ total_passing += 1;
+ } else {
+ failing.push(t);
+ }
+ }
+
+ println!(
+ "REFTEST INFO | {} passing, {} failing",
+ total_passing,
+ failing.len()
+ );
+
+ if !failing.is_empty() {
+ println!("\nReftests with unexpected results:");
+
+ for reftest in &failing {
+ println!("\t{}", reftest);
+ }
+ }
+
+ failing.len()
+ }
+
+ fn run_reftest(&mut self, t: &Reftest) -> bool {
+ let test_name = t.to_string();
+ println!("REFTEST {}", test_name);
+ profile_scope!("wrench reftest", text: &test_name);
+
+ self.wrench
+ .api
+ .send_debug_cmd(
+ DebugCommand::ClearCaches(ClearCache::all())
+ );
+
+ let quality_settings = QualitySettings {
+ force_subpixel_aa_where_possible: t.force_subpixel_aa_where_possible.unwrap_or_default(),
+ };
+
+ self.wrench.set_quality_settings(quality_settings);
+
+ if let Some(max_surface_override) = t.max_surface_override {
+ self.wrench
+ .api
+ .send_debug_cmd(
+ DebugCommand::SetMaximumSurfaceSize(Some(max_surface_override))
+ );
+ }
+
+ let window_size = self.window.get_inner_size();
+ let reference_image = match t.reference.extension().unwrap().to_str().unwrap() {
+ "yaml" => None,
+ "png" => Some(self.load_image(t.reference.as_path(), ImageFormat::Png)),
+ other => panic!("Unknown reftest extension: {}", other),
+ };
+ let test_size = reference_image.as_ref().map_or(window_size, |img| img.size);
+
+ // The reference can be smaller than the window size, in which case
+ // we only compare the intersection.
+ //
+ // Note also that, when we have multiple test scenes in sequence, we
+ // want to test the picture caching machinery. But since picture caching
+ // only takes effect after the result has been the same several frames in
+ // a row, we need to render the scene multiple times.
+ let mut images = vec![];
+ let mut results = vec![];
+
+ match t.op {
+ ReftestOp::Equal | ReftestOp::NotEqual => {
+ // For equality tests, render each test image and store result
+ for filename in t.test.iter() {
+ let output = self.render_yaml(
+ filename,
+ test_size,
+ t.font_render_mode,
+ t.allow_mipmaps,
+ );
+ images.push(output.image);
+ results.push(output.results);
+ }
+ }
+ ReftestOp::Accurate | ReftestOp::Inaccurate => {
+ // For accuracy tests, render the reference yaml at an arbitrary series
+ // of tile sizes, and compare to the reference drawn at normal tile size.
+ let tile_sizes = [
+ DeviceIntSize::new(128, 128),
+ DeviceIntSize::new(256, 256),
+ DeviceIntSize::new(512, 512),
+ ];
+
+ for tile_size in &tile_sizes {
+ self.wrench
+ .api
+ .send_debug_cmd(
+ DebugCommand::SetPictureTileSize(Some(*tile_size))
+ );
+
+ let output = self.render_yaml(
+ &t.reference,
+ test_size,
+ t.font_render_mode,
+ t.allow_mipmaps,
+ );
+ images.push(output.image);
+ results.push(output.results);
+ }
+
+ self.wrench
+ .api
+ .send_debug_cmd(
+ DebugCommand::SetPictureTileSize(None)
+ );
+ }
+ }
+
+ let reference = if let Some(image) = reference_image {
+ let save_all_png = false; // flip to true to update all the tests!
+ if save_all_png {
+ let img = images.last().unwrap();
+ save_flipped(&t.reference, img.data.clone(), img.size);
+ }
+ image
+ } else {
+ let output = self.render_yaml(
+ &t.reference,
+ test_size,
+ t.font_render_mode,
+ t.allow_mipmaps,
+ );
+ output.image
+ };
+
+ if let Some(_) = t.max_surface_override {
+ self.wrench
+ .api
+ .send_debug_cmd(
+ DebugCommand::SetMaximumSurfaceSize(None)
+ );
+ }
+
+ for extra_check in t.extra_checks.iter() {
+ if !extra_check.run(&results) {
+ println!(
+ "REFTEST TEST-UNEXPECTED-FAIL | {} | Failing Check: {:?} | Actual Results: {:?}",
+ t,
+ extra_check,
+ results,
+ );
+ println!("REFTEST TEST-END | {}", t);
+ return false;
+ }
+ }
+
+ match t.op {
+ ReftestOp::Equal => {
+ // Ensure that the final image matches the reference
+ let test = images.pop().unwrap();
+ let comparison = test.compare(&reference);
+ t.check_and_report_equality_failure(
+ comparison,
+ &test,
+ &reference,
+ )
+ }
+ ReftestOp::NotEqual => {
+ // Ensure that the final image *doesn't* match the reference
+ let test = images.pop().unwrap();
+ let comparison = test.compare(&reference);
+ match comparison {
+ ReftestImageComparison::Equal => {
+ t.report_unexpected_equality();
+ false
+ }
+ ReftestImageComparison::NotEqual { .. } => {
+ true
+ }
+ }
+ }
+ ReftestOp::Accurate => {
+ // Ensure that *all* images match the reference
+ for test in images.drain(..) {
+ let comparison = test.compare(&reference);
+
+ if !t.check_and_report_equality_failure(
+ comparison,
+ &test,
+ &reference,
+ ) {
+ return false;
+ }
+ }
+
+ true
+ }
+ ReftestOp::Inaccurate => {
+ // Ensure that at least one of the images doesn't match the reference
+ let all_same = images.iter().all(|image| {
+ match image.compare(&reference) {
+ ReftestImageComparison::Equal => true,
+ ReftestImageComparison::NotEqual { .. } => false,
+ }
+ });
+
+ if all_same {
+ t.report_unexpected_equality();
+ }
+
+ !all_same
+ }
+ }
+ }
+
+ fn load_image(&mut self, filename: &Path, format: ImageFormat) -> ReftestImage {
+ let file = BufReader::new(File::open(filename).unwrap());
+ let img_raw = load_piston_image(file, format).unwrap();
+ let img = img_raw.flipv().to_rgba();
+ let size = img.dimensions();
+ ReftestImage {
+ data: img.into_raw(),
+ size: DeviceIntSize::new(size.0 as i32, size.1 as i32),
+ }
+ }
+
+ fn render_yaml(
+ &mut self,
+ filename: &Path,
+ size: DeviceIntSize,
+ font_render_mode: Option<FontRenderMode>,
+ allow_mipmaps: bool,
+ ) -> YamlRenderOutput {
+ let mut reader = YamlFrameReader::new(filename);
+ reader.set_font_render_mode(font_render_mode);
+ reader.allow_mipmaps(allow_mipmaps);
+ reader.do_frame(self.wrench);
+
+ self.wrench.api.flush_scene_builder();
+
+ // wait for the frame
+ self.rx.recv().unwrap();
+ let results = self.wrench.render();
+
+ let window_size = self.window.get_inner_size();
+ assert!(
+ size.width <= window_size.width &&
+ size.height <= window_size.height,
+ "size={:?} ws={:?}", size, window_size
+ );
+
+ // taking the bottom left sub-rectangle
+ let rect = FramebufferIntRect::from_origin_and_size(
+ FramebufferIntPoint::new(0, window_size.height - size.height),
+ FramebufferIntSize::new(size.width, size.height),
+ );
+ let pixels = self.wrench.renderer.read_pixels_rgba8(rect);
+ self.window.swap_buffers();
+
+ let write_debug_images = false;
+ if write_debug_images {
+ let debug_path = filename.with_extension("yaml.png");
+ save_flipped(debug_path, pixels.clone(), size);
+ }
+
+ reader.deinit(self.wrench);
+
+ YamlRenderOutput {
+ image: ReftestImage { data: pixels, size },
+ results,
+ }
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-13 21:39:49 +0000