#!/usr/bin/perl # # make_tests.pl - generate WPT test cases from the testable statements wiki # # This script assumes that a wiki has testable statement entries # in the format described by the specification at # https://spec-ops.github.io/atta-api/index.html # # usage: make_tests.pl -f file | -w wiki_title | -s spec -d dir use strict; use IO::String ; use JSON ; use MediaWiki::API ; use Getopt::Long; my %specs = ( "aria11" => { title => "ARIA_1.1_Testable_Statements", specURL => "https://www.w3.org/TR/wai-aria11/", dir => "aria11" }, "svg" => { title => "SVG_Accessibility/Testing/Test_Assertions_with_Tables_for_ATTA", specURL => "https://www.w3.org/TR/svg-aam-1.0/", dir => "svg", fragment => '%code%' } ); my @apiNames = qw(UIA MSAA ATK IAccessible2 AXAPI); my $apiNamesRegex = "(" . join("|", @apiNames) . ")"; # the suffix to attach to the automatically generated test case names my $theSuffix = "-manual.html"; # dir is determined based upon the short name of the spec and is defined # by the input or on the command line my $file = undef ; my $spec = undef ; my $wiki_title = undef ; my $dir = undef; my $theSpecFragment = "%code%"; my $preserveWiki = ""; my $fake = 0; my $result = GetOptions( "f|file=s" => \$file, "p=s" => \$preserveWiki, "w|wiki=s" => \$wiki_title, "s|spec=s" => \$spec, "f|fake" => \$fake, "d|dir=s" => \$dir) || usage(); my $wiki_config = { "api_url" => "https://www.w3.org/wiki/api.php" }; my $io ; our $theSpecURL = ""; if ($spec) { print "Processing spec $spec\n"; $wiki_title = $specs{$spec}->{title}; $theSpecURL = $specs{$spec}->{specURL}; if (!$dir) { $dir = "../" . $specs{$spec}->{dir}; } $theSpecFragment = $specs{$spec}->{fragment}; } if (!$dir) { $dir = "../raw"; } if (!-d $dir) { print STDERR "No such directory: $dir\n"; exit 1; } if ($file) { open($io, "<", $file) || die("Failed to open $file: " . $@); } elsif ($wiki_title) { my $MW = MediaWiki::API->new( $wiki_config ); $MW->{config}->{on_error} = \&on_error; sub on_error { print "Error code: " . $MW->{error}->{code} . "\n"; print $MW->{error}->{stacktrace}."\n"; die; } my $page = $MW->get_page( { title => $wiki_title } ); my $theContent = $page->{'*'}; print "Loaded " . length($theContent) . " from $wiki_title\n"; if ($preserveWiki) { if (open(OUTPUT, ">$preserveWiki")) { print OUTPUT $theContent; close OUTPUT; print "Wiki preserved in $preserveWiki\n"; exit 0; } else { print "Failed to create $preserveWiki. Terminating.\n"; exit 1; } } $io = IO::String->new($theContent); } else { usage() ; } # Now let's walk through the content and build a test page for every item # # iterate over the content # my $io ; # open($io, "<", "raw") ; # data structure: # # steps is a list of steps to be performed. # Each step is an object that has a type property and other properties based upon that type. # # Types include: # # 'test' - has a property for each ATAPI for which there are tests # 'attribute' - has a property for the target id, attribute name, and value # 'event' - has a property for the target id and event name my $state = 0; # between items my $theStep = undef; my $current = ""; my $theCode = ""; my $theAttributes = {}; my @steps ; my $theAsserts = {} ; my $theAssertCount = 0; my $theAPI = ""; my $typeRows = 0; my $theType = ""; my $theName = ""; my $theRef = ""; my $lineCounter = 0; my $skipping = 0; our $testNames = {} ; while (<$io>) { if (m//) { last; } $lineCounter++; # look for state if (m/^SpecURL: (.*)$/) { $theSpecURL = $1; $theSpecURL =~ s/^ *//; $theSpecURL =~ s/ *$//; } if ($state == 5 && m/^; \/\/ (.*)/) { # we found another test inside a block # we were in an item; dump it build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ; # print "Finished $current and new subblock $1\n"; $state = 1; $theAttributes = {} ; $theAPI = ""; @steps = (); $theCode = ""; $theAsserts = undef; $theName = ""; } elsif (m/^=== +(.*[^ ]) +===/) { if ($state != 0) { if ($skipping) { print STDERR "Flag on assertion $current; skipping\n"; } else { # we were in an item; dump it build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ; # print "Finished $current\n"; } } $state = 1; $current = $1; $theAttributes = {} ; @steps = (); $theCode = ""; $theAsserts = undef; $theAPI = ""; $theName = ""; if ($current =~ m/\(/) { # there is a paren in the name -skip it $skipping = 1; } else { $skipping = 0; } } if ($state == 1) { if (m/
/) {
      # we are now in the code block
      $state = 2;
      next;
    } elsif (m/==== +(.*) +====/) {
      # we are in some other block
      $theName = lc($1);
      $theAttributes->{$theName} = "";
      next;
    }
    if (m/^Reference: +(.*)$/) {
      $theAttributes->{reference} = $theSpecURL . "#" . $1;
    } elsif ($theName ne "") {
      # accumulate whatever was in the block under the data for it
      chomp();
      $theAttributes->{$theName} .= $_;
    } elsif (m/TODO/) {
      $state = 0;
    }
  }

  if ($state == 2) {
    if (m/<\/pre>/) {
      # we are done with the code block
      $state = 3;
    } else  {
      if (m/^\s/ && !m/if given/) {
        # trim any trailing whitespace
        $theCode =~ s/ +$//;
        $theCode =~ s/\t/ /g;
        $theCode .= $_;
        # In MediaWiki, to display & symbol escapes as literal text, one
        # must use "&&" for the "&" character. We need to undo that.
        $theCode =~ s/&(\S)/&$1/g;
      }
    }
  } elsif ($state == 3) {
    # look for a table
    if (m/^\{\|/) {
      # table started
      $state = 4;
    }
  } elsif ($state == 4) {
    if (m/^\|-/) {
      if ($theAPI
        && exists($theAsserts->{$theAPI}->[$theAssertCount])
        && scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) {
        $theAssertCount++;
      }
      # start of a table row
      if ($theType ne "" && $typeRows) {
        # print qq($theType typeRows was $typeRows\n);
        # we are still processing items for a type
        $typeRows--;
        # populate the first cell
        $theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ;
      } else {
        $theType = "";
      }
    } elsif (m/^\|\}/) {
      # ran out of table
      $state = 5;
    # adding processing for additional block types
    # a colspan followed by a keyword triggers a start
    # so |colspan=5|element triggers a new collection
    # |colspan=5|attribute triggers the setting of an attribute
    } elsif (m/^\|colspan="*([0-9])"*\|([^ ]+) (.*)$/) {
      my $type = $2;
      my $params = $3;

      my $obj = {} ;
      if ($type eq "attribute") {
        if ($params =~ m/([^:]+):([^ ]+) +(.*)$/) {
          $obj = {
            type => $type,
            element => $1,
            attribute => $2,
            value => $3
          };
          $theStep = undef;
          push(@steps, $obj);
        } else {
          print STDERR "Malformed attribute instruction at line $lineCounter: " . $_ . "\n";
        }
      } elsif ($type eq "event") {
        if ($params =~ m/([^:]+):([^ ]+).*$/) {
          $obj = {
            type => $type,
            element => $1,
            value => $2
          };
          $theStep = undef;
          push(@steps, $obj);
        } else {
          print STDERR "Malformed event instruction at line $lineCounter: " . $_ . "\n";
        }
      } elsif ($type eq "element") {
        $obj = {
          type => "test",
          element => $3
        };
        push(@steps, $obj);
        $theStep = scalar(@steps) - 1;
        $theAsserts = $steps[$theStep];
      } else {
        print STDERR "Unknown operation type: $type at line " . $lineCounter . "; skipping.\n";
      }
    } elsif (m/($apiNamesRegex)$/) {
      my $theString = $1;
      $theString =~ s/ +$//;
      $theString =~ s/^ +//;
      if ($theString eq "IA2") {
        $theString = "IAccessible2" ;
      }
      my $rows = 1;
      if (m/^\|rowspan="*([0-9])"*\|(.*)$/) {
        $rows = $1
      }
      if (grep { $_ eq $theString } @apiNames) {
        # we found an API name - were we already processing assertions?
        if (!$theAsserts) {
          # nope - now what?
          $theAsserts = {
            type => "test",
            element => "test"
          };
          push(@steps, $theAsserts);
        }
        $theAssertCount = 0;
        # this is a new API section
        $theAPI = $theString ;
        $theAsserts->{$theAPI} = [ [] ] ;
        $theType = "";
      } else {
        # this is a multi-row type
        $theType = $theString;
        $typeRows = $rows;
        # print qq(Found multi-row $theString for $theAPI with $typeRows rows\n);
        $typeRows--;
        # populate the first cell
        if ($theAPI
          && exists($theAsserts->{$theAPI}->[$theAssertCount])
          && scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) {
          $theAssertCount++;
        }
        $theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ;
      }
    } elsif (m/^\|(.*)$/) {
      my $item = $1;
      $item =~ s/^ *//;
      $item =~ s/ *$//;
      $item =~ s/^['"]//;
      $item =~ s/['"]$//;
      # add into the data structure for the API
      if (!exists $theAsserts->{$theAPI}->[$theAssertCount]) {
        $theAsserts->{$theAPI}->[$theAssertCount] = [ $item ] ;
      } else {
        push(@{$theAsserts->{$theAPI}->[$theAssertCount]}, $item);
      }
    }
  }
};

if ($state != 0) {
  build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ;
  print "Finished $current\n";
}

exit 0;

# build_test
#
# create a test file
#
# attempts to create unique test names

sub build_test() {
  my $title = shift ;
  my $attrs = shift ;
  my $code = shift ;
  my $steps = shift;
  my $frag = shift ;

  if ($title eq "") {
    print STDERR "No name provided!";
    return;
  }

  if ($frag ne "") {
    $frag =~ s/%code%/$code/;
    $code = $frag;
  }

  $code =~ s/ +$//m;
  $code =~ s/\t/ /g;

  my $title_reference = $title;

  if ($code eq "") {
    print STDERR "No code for $title; skipping.\n";
    return;
  }
  if ( $steps eq {}) {
    print STDERR "No assertions for $title; skipping.\n";
    return;
  }

  my $testDef =
  { "title" => $title,
    "steps" => []
  };
  my $stepCount = 0;
  foreach my $asserts (@$steps) {
    $stepCount++;
    my $step =
      {
        "type" => $asserts->{"type"},
        "title"=> "step " . $stepCount,
      };

    if ($asserts->{type} eq "test") {
      # everything in the block is about testing an element
      $step->{"element"} = ( $asserts->{"element"} || "test" );

      my $tests = {};
      if ($fake) {
        $tests->{"WAIFAKE"} = [ [ "property", "role", "is", "ROLE_TABLE_CELL" ], [ "property", "interfaces", "contains", "TableCell" ] ];
      }
      foreach my $name (@apiNames) {
        if (exists $asserts->{$name} && scalar(@{$asserts->{$name}})) {
          $tests->{$name} = $asserts->{$name};
        }
      };

      $step->{test} = $tests;

    } elsif ($asserts->{type} eq "attribute") {
      $step->{type} = "attribute";
      $step->{element} = $asserts->{"element"};
      $step->{attribute} = $asserts->{"attribute"};
      $step->{value} = $asserts->{value};
    } elsif ($asserts->{type} eq "event") {
      $step->{type} = "event";
      $step->{element} = $asserts->{"element"};
      $step->{event} = $asserts->{value};
    } else {
      print STDERR "Invalid step type: " . $asserts->{type} . "\n";
      next;
    }
    push(@{$testDef->{steps}}, $step);
  }


  # populate the rest of the test definition

  if (scalar(keys(%$attrs))) {
    while (my $key = each(%$attrs)) {
      # print "Copying $key \n";
      $testDef->{$key} = $attrs->{$key};
    }
  }

  if (exists $attrs->{reference}) {
    $title_reference = "" . $title_reference . "" ;
  }

  my $testDef_json = to_json($testDef, { canonical => 1, pretty => 1, utf8 => 1});

  my $fileName = $title;
  $fileName =~ s/\s*$//;
  $fileName =~ s/\///g;
  $fileName =~ s/\s+/_/g;
  $fileName =~ s/[,=:]/_/g;
  $fileName =~ s/['"]//g;

  my $count = 2;
  if ($testNames->{$fileName}) {
    while (exists $testNames->{$fileName . "_$count"}) {
      $count++;
    }
    $fileName .= "_$count";
  }

  $fileName = lc($fileName);

  $testNames->{$fileName} = 1;

  $fileName .= $theSuffix;

  my $template = qq(

  
    $title
    
    
    
    
    
    
  
  
  

This test examines the ARIA properties for $title_reference.

$code
); my $file ; if (open($file, ">", "$dir/$fileName")) { print $file $template; print $file "\n"; close $file; } else { print STDERR qq(Failed to create file "$dir/$fileName" $!\n); } return; } sub usage() { print STDERR q(usage: make_tests.pl -f file | -w wiki_title | -s spec [-n -v -d dir ] -s specname - the name of a spec known to the system -w wiki_title - the TITLE of a wiki page with testable statements -f file - the file from which to read -n - do nothing -v - be verbose -d dir - put generated tests in directory dir ); exit 1; } # vim: ts=2 sw=2 ai: