#!/usr/bin/perl # # make_tests.pl - generate WPT test cases from the testable statements wiki # # This script assumes that a wiki has testable statement entries # in the format described by the specification at # https://spec-ops.github.io/atta-api/index.html # # usage: make_tests.pl -f file | -w wiki_title | -s spec -d dir use strict; use IO::String ; use JSON ; use MediaWiki::API ; use Getopt::Long; my %specs = ( "aria11" => { title => "ARIA_1.1_Testable_Statements", specURL => "https://www.w3.org/TR/wai-aria11/", dir => "aria11" }, "svg" => { title => "SVG_Accessibility/Testing/Test_Assertions_with_Tables_for_ATTA", specURL => "https://www.w3.org/TR/svg-aam-1.0/", dir => "svg", fragment => '' } ); my @apiNames = qw(UIA MSAA ATK IAccessible2 AXAPI); my $apiNamesRegex = "(" . join("|", @apiNames) . ")"; # the suffix to attach to the automatically generated test case names my $theSuffix = "-manual.html"; # dir is determined based upon the short name of the spec and is defined # by the input or on the command line my $file = undef ; my $spec = undef ; my $wiki_title = undef ; my $dir = undef; my $theSpecFragment = "%code%"; my $preserveWiki = ""; my $fake = 0; my $result = GetOptions( "f|file=s" => \$file, "p=s" => \$preserveWiki, "w|wiki=s" => \$wiki_title, "s|spec=s" => \$spec, "f|fake" => \$fake, "d|dir=s" => \$dir) || usage(); my $wiki_config = { "api_url" => "https://www.w3.org/wiki/api.php" }; my $io ; our $theSpecURL = ""; if ($spec) { print "Processing spec $spec\n"; $wiki_title = $specs{$spec}->{title}; $theSpecURL = $specs{$spec}->{specURL}; if (!$dir) { $dir = "../" . $specs{$spec}->{dir}; } $theSpecFragment = $specs{$spec}->{fragment}; } if (!$dir) { $dir = "../raw"; } if (!-d $dir) { print STDERR "No such directory: $dir\n"; exit 1; } if ($file) { open($io, "<", $file) || die("Failed to open $file: " . $@); } elsif ($wiki_title) { my $MW = MediaWiki::API->new( $wiki_config ); $MW->{config}->{on_error} = \&on_error; sub on_error { print "Error code: " . $MW->{error}->{code} . "\n"; print $MW->{error}->{stacktrace}."\n"; die; } my $page = $MW->get_page( { title => $wiki_title } ); my $theContent = $page->{'*'}; print "Loaded " . length($theContent) . " from $wiki_title\n"; if ($preserveWiki) { if (open(OUTPUT, ">$preserveWiki")) { print OUTPUT $theContent; close OUTPUT; print "Wiki preserved in $preserveWiki\n"; exit 0; } else { print "Failed to create $preserveWiki. Terminating.\n"; exit 1; } } $io = IO::String->new($theContent); } else { usage() ; } # Now let's walk through the content and build a test page for every item # # iterate over the content # my $io ; # open($io, "<", "raw") ; # data structure: # # steps is a list of steps to be performed. # Each step is an object that has a type property and other properties based upon that type. # # Types include: # # 'test' - has a property for each ATAPI for which there are tests # 'attribute' - has a property for the target id, attribute name, and value # 'event' - has a property for the target id and event name my $state = 0; # between items my $theStep = undef; my $current = ""; my $theCode = ""; my $theAttributes = {}; my @steps ; my $theAsserts = {} ; my $theAssertCount = 0; my $theAPI = ""; my $typeRows = 0; my $theType = ""; my $theName = ""; my $theRef = ""; my $lineCounter = 0; my $skipping = 0; our $testNames = {} ; while (<$io>) { if (m//) { last; } $lineCounter++; # look for state if (m/^SpecURL: (.*)$/) { $theSpecURL = $1; $theSpecURL =~ s/^ *//; $theSpecURL =~ s/ *$//; } if ($state == 5 && m/^; \/\/ (.*)/) { # we found another test inside a block # we were in an item; dump it build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ; # print "Finished $current and new subblock $1\n"; $state = 1; $theAttributes = {} ; $theAPI = ""; @steps = (); $theCode = ""; $theAsserts = undef; $theName = ""; } elsif (m/^=== +(.*[^ ]) +===/) { if ($state != 0) { if ($skipping) { print STDERR "Flag on assertion $current; skipping\n"; } else { # we were in an item; dump it build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ; # print "Finished $current\n"; } } $state = 1; $current = $1; $theAttributes = {} ; @steps = (); $theCode = ""; $theAsserts = undef; $theAPI = ""; $theName = ""; if ($current =~ m/\(/) { # there is a paren in the name -skip it $skipping = 1; } else { $skipping = 0; } } if ($state == 1) { if (m/
/) { # we are now in the code block $state = 2; next; } elsif (m/==== +(.*) +====/) { # we are in some other block $theName = lc($1); $theAttributes->{$theName} = ""; next; } if (m/^Reference: +(.*)$/) { $theAttributes->{reference} = $theSpecURL . "#" . $1; } elsif ($theName ne "") { # accumulate whatever was in the block under the data for it chomp(); $theAttributes->{$theName} .= $_; } elsif (m/TODO/) { $state = 0; } } if ($state == 2) { if (m/<\/pre>/) { # we are done with the code block $state = 3; } else { if (m/^\s/ && !m/if given/) { # trim any trailing whitespace $theCode =~ s/ +$//; $theCode =~ s/\t/ /g; $theCode .= $_; # In MediaWiki, to display & symbol escapes as literal text, one # must use "&&" for the "&" character. We need to undo that. $theCode =~ s/&(\S)/&$1/g; } } } elsif ($state == 3) { # look for a table if (m/^\{\|/) { # table started $state = 4; } } elsif ($state == 4) { if (m/^\|-/) { if ($theAPI && exists($theAsserts->{$theAPI}->[$theAssertCount]) && scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) { $theAssertCount++; } # start of a table row if ($theType ne "" && $typeRows) { # print qq($theType typeRows was $typeRows\n); # we are still processing items for a type $typeRows--; # populate the first cell $theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ; } else { $theType = ""; } } elsif (m/^\|\}/) { # ran out of table $state = 5; # adding processing for additional block types # a colspan followed by a keyword triggers a start # so |colspan=5|element triggers a new collection # |colspan=5|attribute triggers the setting of an attribute } elsif (m/^\|colspan="*([0-9])"*\|([^ ]+) (.*)$/) { my $type = $2; my $params = $3; my $obj = {} ; if ($type eq "attribute") { if ($params =~ m/([^:]+):([^ ]+) +(.*)$/) { $obj = { type => $type, element => $1, attribute => $2, value => $3 }; $theStep = undef; push(@steps, $obj); } else { print STDERR "Malformed attribute instruction at line $lineCounter: " . $_ . "\n"; } } elsif ($type eq "event") { if ($params =~ m/([^:]+):([^ ]+).*$/) { $obj = { type => $type, element => $1, value => $2 }; $theStep = undef; push(@steps, $obj); } else { print STDERR "Malformed event instruction at line $lineCounter: " . $_ . "\n"; } } elsif ($type eq "element") { $obj = { type => "test", element => $3 }; push(@steps, $obj); $theStep = scalar(@steps) - 1; $theAsserts = $steps[$theStep]; } else { print STDERR "Unknown operation type: $type at line " . $lineCounter . "; skipping.\n"; } } elsif (m/($apiNamesRegex)$/) { my $theString = $1; $theString =~ s/ +$//; $theString =~ s/^ +//; if ($theString eq "IA2") { $theString = "IAccessible2" ; } my $rows = 1; if (m/^\|rowspan="*([0-9])"*\|(.*)$/) { $rows = $1 } if (grep { $_ eq $theString } @apiNames) { # we found an API name - were we already processing assertions? if (!$theAsserts) { # nope - now what? $theAsserts = { type => "test", element => "test" }; push(@steps, $theAsserts); } $theAssertCount = 0; # this is a new API section $theAPI = $theString ; $theAsserts->{$theAPI} = [ [] ] ; $theType = ""; } else { # this is a multi-row type $theType = $theString; $typeRows = $rows; # print qq(Found multi-row $theString for $theAPI with $typeRows rows\n); $typeRows--; # populate the first cell if ($theAPI && exists($theAsserts->{$theAPI}->[$theAssertCount]) && scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) { $theAssertCount++; } $theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ; } } elsif (m/^\|(.*)$/) { my $item = $1; $item =~ s/^ *//; $item =~ s/ *$//; $item =~ s/^['"]//; $item =~ s/['"]$//; # add into the data structure for the API if (!exists $theAsserts->{$theAPI}->[$theAssertCount]) { $theAsserts->{$theAPI}->[$theAssertCount] = [ $item ] ; } else { push(@{$theAsserts->{$theAPI}->[$theAssertCount]}, $item); } } } }; if ($state != 0) { build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ; print "Finished $current\n"; } exit 0; # build_test # # create a test file # # attempts to create unique test names sub build_test() { my $title = shift ; my $attrs = shift ; my $code = shift ; my $steps = shift; my $frag = shift ; if ($title eq "") { print STDERR "No name provided!"; return; } if ($frag ne "") { $frag =~ s/%code%/$code/; $code = $frag; } $code =~ s/ +$//m; $code =~ s/\t/ /g; my $title_reference = $title; if ($code eq "") { print STDERR "No code for $title; skipping.\n"; return; } if ( $steps eq {}) { print STDERR "No assertions for $title; skipping.\n"; return; } my $testDef = { "title" => $title, "steps" => [] }; my $stepCount = 0; foreach my $asserts (@$steps) { $stepCount++; my $step = { "type" => $asserts->{"type"}, "title"=> "step " . $stepCount, }; if ($asserts->{type} eq "test") { # everything in the block is about testing an element $step->{"element"} = ( $asserts->{"element"} || "test" ); my $tests = {}; if ($fake) { $tests->{"WAIFAKE"} = [ [ "property", "role", "is", "ROLE_TABLE_CELL" ], [ "property", "interfaces", "contains", "TableCell" ] ]; } foreach my $name (@apiNames) { if (exists $asserts->{$name} && scalar(@{$asserts->{$name}})) { $tests->{$name} = $asserts->{$name}; } }; $step->{test} = $tests; } elsif ($asserts->{type} eq "attribute") { $step->{type} = "attribute"; $step->{element} = $asserts->{"element"}; $step->{attribute} = $asserts->{"attribute"}; $step->{value} = $asserts->{value}; } elsif ($asserts->{type} eq "event") { $step->{type} = "event"; $step->{element} = $asserts->{"element"}; $step->{event} = $asserts->{value}; } else { print STDERR "Invalid step type: " . $asserts->{type} . "\n"; next; } push(@{$testDef->{steps}}, $step); } # populate the rest of the test definition if (scalar(keys(%$attrs))) { while (my $key = each(%$attrs)) { # print "Copying $key \n"; $testDef->{$key} = $attrs->{$key}; } } if (exists $attrs->{reference}) { $title_reference = "" . $title_reference . "" ; } my $testDef_json = to_json($testDef, { canonical => 1, pretty => 1, utf8 => 1}); my $fileName = $title; $fileName =~ s/\s*$//; $fileName =~ s/\///g; $fileName =~ s/\s+/_/g; $fileName =~ s/[,=:]/_/g; $fileName =~ s/['"]//g; my $count = 2; if ($testNames->{$fileName}) { while (exists $testNames->{$fileName . "_$count"}) { $count++; } $fileName .= "_$count"; } $fileName = lc($fileName); $testNames->{$fileName} = 1; $fileName .= $theSuffix; my $template = qq($title This test examines the ARIA properties for $title_reference.
$code ); my $file ; if (open($file, ">", "$dir/$fileName")) { print $file $template; print $file "\n"; close $file; } else { print STDERR qq(Failed to create file "$dir/$fileName" $!\n); } return; } sub usage() { print STDERR q(usage: make_tests.pl -f file | -w wiki_title | -s spec [-n -v -d dir ] -s specname - the name of a spec known to the system -w wiki_title - the TITLE of a wiki page with testable statements -f file - the file from which to read -n - do nothing -v - be verbose -d dir - put generated tests in directory dir ); exit 1; } # vim: ts=2 sw=2 ai: